This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e9547f13c1 Test Classes Cleanup For Setters of IndexLoadingConfig
#14106 (#14140)
e9547f13c1 is described below
commit e9547f13c1ebf5efa46bd72fe5feb39824c71d85
Author: Chaitanya Deepthi <[email protected]>
AuthorDate: Wed Oct 2 12:38:16 2024 -0700
Test Classes Cleanup For Setters of IndexLoadingConfig #14106 (#14140)
---
.../pinot/queries/FilteredAggregationsTest.java | 31 +++++++----------
.../apache/pinot/queries/H3IndexQueriesTest.java | 7 +---
.../org/apache/pinot/queries/RangeQueriesTest.java | 40 +++++++++++++++-------
.../segment/creator/OnHeapDictionariesTest.java | 11 ++----
4 files changed, 44 insertions(+), 45 deletions(-)
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/FilteredAggregationsTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/FilteredAggregationsTest.java
index b59ff81c6b..02a9a05395 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/FilteredAggregationsTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/FilteredAggregationsTest.java
@@ -22,10 +22,8 @@ import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
import java.util.Random;
-import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.pinot.common.response.broker.ResultTable;
@@ -62,6 +60,16 @@ public class FilteredAggregationsTest extends
BaseQueriesTest {
private static final String BOOLEAN_COL_NAME = "BOOLEAN_COL";
private static final String STRING_COL_NAME = "STRING_COL";
private static final Integer NUM_ROWS = 30000;
+ private static final Schema SCHEMA = new
Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ .addSingleValueDimension(NO_INDEX_INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(STATIC_INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(BOOLEAN_COL_NAME, FieldSpec.DataType.BOOLEAN)
+ .addSingleValueDimension(STRING_COL_NAME, FieldSpec.DataType.STRING)
+ .addMetric(INT_COL_NAME, FieldSpec.DataType.INT).build();
+ private static final List<FieldConfig> FIELD_CONFIGS = new ArrayList<>();
+ private static final TableConfig TABLE_CONFIG = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
+
.setInvertedIndexColumns(Collections.singletonList(INT_COL_NAME)).setRangeIndexColumns(List.of(INT_COL_NAME))
+ .setFieldConfigList(FIELD_CONFIGS).build();
private IndexSegment _indexSegment;
private List<IndexSegment> _indexSegments;
@@ -88,13 +96,8 @@ public class FilteredAggregationsTest extends
BaseQueriesTest {
buildSegment(FIRST_SEGMENT_NAME);
buildSegment(SECOND_SEGMENT_NAME);
- IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig();
- Set<String> invertedIndexCols = new HashSet<>();
- invertedIndexCols.add(INT_COL_NAME);
-
- indexLoadingConfig.setInvertedIndexColumns(invertedIndexCols);
- indexLoadingConfig.setRangeIndexColumns(invertedIndexCols);
+ IndexLoadingConfig indexLoadingConfig = new
IndexLoadingConfig(TABLE_CONFIG, SCHEMA);
ImmutableSegment firstImmutableSegment =
ImmutableSegmentLoader.load(new File(INDEX_DIR, FIRST_SEGMENT_NAME),
indexLoadingConfig);
ImmutableSegment secondImmutableSegment =
@@ -127,17 +130,7 @@ public class FilteredAggregationsTest extends
BaseQueriesTest {
private void buildSegment(String segmentName)
throws Exception {
List<GenericRow> rows = createTestData();
- List<FieldConfig> fieldConfigs = new ArrayList<>();
-
- TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
-
.setInvertedIndexColumns(Collections.singletonList(INT_COL_NAME)).setFieldConfigList(fieldConfigs).build();
- Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
- .addSingleValueDimension(NO_INDEX_INT_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(STATIC_INT_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(BOOLEAN_COL_NAME, FieldSpec.DataType.BOOLEAN)
- .addSingleValueDimension(STRING_COL_NAME, FieldSpec.DataType.STRING)
- .addMetric(INT_COL_NAME, FieldSpec.DataType.INT).build();
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(TABLE_CONFIG,
SCHEMA);
config.setOutDir(INDEX_DIR.getPath());
config.setTableName(TABLE_NAME);
config.setSegmentName(segmentName);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/H3IndexQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/H3IndexQueriesTest.java
index e21a252f00..f8ff4661bb 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/H3IndexQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/H3IndexQueriesTest.java
@@ -19,7 +19,6 @@
package org.apache.pinot.queries;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -39,7 +38,6 @@ import
org.apache.pinot.segment.local.utils.GeometrySerializer;
import org.apache.pinot.segment.local.utils.GeometryUtils;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
-import org.apache.pinot.segment.spi.index.creator.H3IndexConfig;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
@@ -111,10 +109,7 @@ public class H3IndexQueriesTest extends BaseQueriesTest {
driver.init(segmentGeneratorConfig, new GenericRowRecordReader(records));
driver.build();
- IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig();
- indexLoadingConfig.setH3IndexConfigs(ImmutableMap
- .of(H3_INDEX_COLUMN, new H3IndexConfig(H3_INDEX_PROPERTIES),
H3_INDEX_GEOMETRY_COLUMN,
- new H3IndexConfig(H3_INDEX_PROPERTIES)));
+ IndexLoadingConfig indexLoadingConfig = new
IndexLoadingConfig(TABLE_CONFIG, SCHEMA);
_indexSegment = ImmutableSegmentLoader.load(new File(INDEX_DIR,
SEGMENT_NAME), indexLoadingConfig);
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
index 86ac43379b..4e64ce1ca9 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
@@ -24,6 +24,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
+import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.core.common.Operator;
import org.apache.pinot.core.operator.query.FastFilteredCountOperator;
@@ -93,6 +94,9 @@ public class RangeQueriesTest extends BaseQueriesTest {
return _indexSegments;
}
+ private Set<String> _noDictionaryColumns;
+ private Set<String> _rangeIndexColumns;
+
@BeforeClass
public void setUp()
throws Exception {
@@ -108,7 +112,6 @@ public class RangeQueriesTest extends BaseQueriesTest {
record.putValue(RAW_DOUBLE_COL, (double) intValue);
records.add(record);
}
-
SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(TABLE_CONFIG, SCHEMA);
segmentGeneratorConfig.setTableName(RAW_TABLE_NAME);
segmentGeneratorConfig.setSegmentName(SEGMENT_NAME);
@@ -118,9 +121,10 @@ public class RangeQueriesTest extends BaseQueriesTest {
driver.init(segmentGeneratorConfig, new GenericRowRecordReader(records));
driver.build();
- IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig();
- indexLoadingConfig.setRangeIndexColumns(
- new HashSet<>(Arrays.asList(DICTIONARIZED_INT_COL, RAW_INT_COL,
RAW_LONG_COL, RAW_FLOAT_COL, RAW_DOUBLE_COL)));
+ _noDictionaryColumns = new HashSet<>(Arrays.asList(RAW_INT_COL,
RAW_LONG_COL, RAW_FLOAT_COL, RAW_DOUBLE_COL));
+ _rangeIndexColumns =
+ new HashSet<>(Arrays.asList(DICTIONARIZED_INT_COL, RAW_INT_COL,
RAW_LONG_COL, RAW_FLOAT_COL, RAW_DOUBLE_COL));
+ IndexLoadingConfig indexLoadingConfig = new
IndexLoadingConfig(TABLE_CONFIG, SCHEMA);
ImmutableSegment immutableSegment =
ImmutableSegmentLoader.load(new File(INDEX_DIR, SEGMENT_NAME),
indexLoadingConfig);
@@ -128,6 +132,16 @@ public class RangeQueriesTest extends BaseQueriesTest {
_indexSegments = Arrays.asList(immutableSegment, immutableSegment);
}
+ private IndexLoadingConfig createIndexLoadingConfig() {
+ return new IndexLoadingConfig(createTableConfig(), SCHEMA);
+ }
+
+ private TableConfig createTableConfig() {
+ return new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setNoDictionaryColumns(new ArrayList<>(_noDictionaryColumns))
+ .setRangeIndexColumns(new ArrayList<>(_rangeIndexColumns)).build();
+ }
+
@DataProvider
public static Object[][] selectionTestCases() {
//@formatter:off
@@ -300,8 +314,9 @@ public class RangeQueriesTest extends BaseQueriesTest {
public void testSelectionOverRangeFilterAfterReload(String query, int min,
int max, boolean inclusive)
throws Exception {
// Enable dictionary on RAW_INT_COL and reload the segment.
- IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig(null,
TABLE_CONFIG);
- indexLoadingConfig.removeNoDictionaryColumns(RAW_INT_COL);
+ _noDictionaryColumns.remove(RAW_INT_COL);
+ IndexLoadingConfig indexLoadingConfig = createIndexLoadingConfig();
+
File indexDir = new File(INDEX_DIR, SEGMENT_NAME);
ImmutableSegment immutableSegment = reloadSegment(indexDir,
indexLoadingConfig, SCHEMA);
_indexSegment = immutableSegment;
@@ -315,9 +330,9 @@ public class RangeQueriesTest extends BaseQueriesTest {
assertTrue(inclusive ? value <= max : value < max);
}
+ _noDictionaryColumns.remove(RAW_DOUBLE_COL);
// Enable dictionary on RAW_DOUBLE_COL and reload the segment.
- indexLoadingConfig = new IndexLoadingConfig(null, TABLE_CONFIG);
- indexLoadingConfig.removeNoDictionaryColumns(RAW_DOUBLE_COL);
+ indexLoadingConfig = createIndexLoadingConfig();
indexDir = new File(INDEX_DIR, SEGMENT_NAME);
immutableSegment = reloadSegment(indexDir, indexLoadingConfig, SCHEMA);
_indexSegment = immutableSegment;
@@ -346,8 +361,9 @@ public class RangeQueriesTest extends BaseQueriesTest {
public void testCountOverRangeFilterAfterReload(String query, int
expectedCount)
throws Exception {
// Enable dictionary on RAW_LONG_COL and reload the segment.
- IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig(null,
TABLE_CONFIG);
- indexLoadingConfig.removeNoDictionaryColumns(RAW_LONG_COL);
+ _noDictionaryColumns.remove(RAW_LONG_COL);
+
+ IndexLoadingConfig indexLoadingConfig = createIndexLoadingConfig();
File indexDir = new File(INDEX_DIR, SEGMENT_NAME);
ImmutableSegment immutableSegment = reloadSegment(indexDir,
indexLoadingConfig, SCHEMA);
_indexSegment = immutableSegment;
@@ -361,8 +377,8 @@ public class RangeQueriesTest extends BaseQueriesTest {
assertEquals(((Number) aggregationResult.get(0)).intValue(),
expectedCount, query);
// Enable dictionary on RAW_FLOAT_COL and reload the segment.
- indexLoadingConfig = new IndexLoadingConfig(null, TABLE_CONFIG);
- indexLoadingConfig.removeNoDictionaryColumns(RAW_FLOAT_COL);
+ _noDictionaryColumns.remove(RAW_FLOAT_COL);
+ indexLoadingConfig = createIndexLoadingConfig();
immutableSegment = reloadSegment(indexDir, indexLoadingConfig, SCHEMA);
_indexSegment = immutableSegment;
_indexSegments = Arrays.asList(immutableSegment, immutableSegment);
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/OnHeapDictionariesTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/OnHeapDictionariesTest.java
index a3701d774c..e455cf1041 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/OnHeapDictionariesTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/OnHeapDictionariesTest.java
@@ -21,9 +21,7 @@ package org.apache.pinot.segment.local.segment.creator;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Random;
import org.apache.commons.io.FileUtils;
@@ -79,17 +77,14 @@ public class OnHeapDictionariesTest {
throws Exception {
Schema schema = buildSchema();
- TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setOnHeapDictionaryColumns(
+ List.of(INT_COLUMN, LONG_COLUMN, FLOAT_COLUMN, DOUBLE_COLUMN,
STRING_COLUMN)).setTableName("test").build();
buildSegment(SEGMENT_DIR_NAME, SEGMENT_NAME, tableConfig, schema);
- IndexLoadingConfig loadingConfig = new IndexLoadingConfig();
+ IndexLoadingConfig loadingConfig = new IndexLoadingConfig(tableConfig,
schema);
loadingConfig.setReadMode(ReadMode.mmap);
loadingConfig.setSegmentVersion(SegmentVersion.v3);
_offHeapSegment = ImmutableSegmentLoader.load(new File(SEGMENT_DIR_NAME,
SEGMENT_NAME), loadingConfig);
-
- loadingConfig.setOnHeapDictionaryColumns(new HashSet<>(
- Arrays.asList(new String[]{INT_COLUMN, LONG_COLUMN, FLOAT_COLUMN,
DOUBLE_COLUMN, STRING_COLUMN})));
-
_onHeapSegment = ImmutableSegmentLoader.load(new File(SEGMENT_DIR_NAME,
SEGMENT_NAME), loadingConfig);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
