This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9ef4beaee6 Remove setters dependency of IndexLoadingConfig class in
Test Classes (#14201)
9ef4beaee6 is described below
commit 9ef4beaee64617238770cd2b38ab9bc93ec6acb9
Author: Chaitanya Deepthi <[email protected]>
AuthorDate: Thu Oct 10 12:30:54 2024 -0700
Remove setters dependency of IndexLoadingConfig class in Test Classes
(#14201)
---
.../perf/BenchmarkColumnValueSegmentPruner.java | 63 ++++++++--------------
.../perf/BenchmarkNativeAndLuceneBasedLike.java | 25 ++++-----
.../apache/pinot/perf/BenchmarkOrderByQueries.java | 55 +++++++++----------
.../org/apache/pinot/perf/BenchmarkQueries.java | 53 ++++++++----------
.../pinot/perf/StringDictionaryPerfTest.java | 17 +++---
5 files changed, 85 insertions(+), 128 deletions(-)
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkColumnValueSegmentPruner.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkColumnValueSegmentPruner.java
index d73da4bbba..f261c6231b 100644
---
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkColumnValueSegmentPruner.java
+++
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkColumnValueSegmentPruner.java
@@ -23,10 +23,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.LongSupplier;
@@ -43,7 +41,6 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import
org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair;
-import org.apache.pinot.spi.config.table.BloomFilterConfig;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.StarTreeIndexConfig;
import org.apache.pinot.spi.config.table.TableConfig;
@@ -79,7 +76,6 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
@Measurement(iterations = 5, time = 2)
@State(Scope.Benchmark)
public class BenchmarkColumnValueSegmentPruner {
-
public static final String QUERY_1 = "SELECT * FROM MyTable WHERE SORTED_COL
IN (1, 2, 3, 4)";
@Param({"10"})
@@ -113,26 +109,32 @@ public class BenchmarkColumnValueSegmentPruner {
private static final String NO_INDEX_INT_COL_NAME = "NO_INDEX_INT_COL";
private static final String NO_INDEX_STRING_COL = "NO_INDEX_STRING_COL";
private static final String LOW_CARDINALITY_STRING_COL =
"LOW_CARDINALITY_STRING_COL";
-
+ private static final List<FieldConfig> FIELD_CONFIGS = new ArrayList<>();
+ private static final TableConfig TABLE_CONFIG = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
+ .setInvertedIndexColumns(List.of(INT_COL_NAME,
LOW_CARDINALITY_STRING_COL)).setFieldConfigList(FIELD_CONFIGS)
+ .setNoDictionaryColumns(List.of(RAW_INT_COL_NAME,
RAW_STRING_COL_NAME)).setSortedColumn(SORTED_COL_NAME)
+
.setVarLengthDictionaryColumns(Collections.singletonList(SORTED_COL_NAME))
+ .setRangeIndexColumns(Arrays.asList(INT_COL_NAME,
LOW_CARDINALITY_STRING_COL))
+
.setBloomFilterColumns(Collections.singletonList(SORTED_COL_NAME)).setStarTreeIndexConfigs(
+ Collections.singletonList(new
StarTreeIndexConfig(List.of(SORTED_COL_NAME, INT_COL_NAME), null,
+ Collections.singletonList(
+ new
AggregationFunctionColumnPair(AggregationFunctionType.SUM,
RAW_INT_COL_NAME).toColumnName()),
+ null, Integer.MAX_VALUE))).build();
+ private static final Schema SCHEMA = new
Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ .addSingleValueDimension(SORTED_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(NO_INDEX_INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(RAW_INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(RAW_STRING_COL_NAME, FieldSpec.DataType.STRING)
+ .addSingleValueDimension(NO_INDEX_STRING_COL, FieldSpec.DataType.STRING)
+ .addSingleValueDimension(LOW_CARDINALITY_STRING_COL,
FieldSpec.DataType.STRING).build();
@Setup
public void setUp()
throws Exception {
_supplier = Distribution.createLongSupplier(42, _scenario);
FileUtils.deleteQuietly(INDEX_DIR);
-
- IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig();
-
- Set<String> invertedIndexCols = new HashSet<>();
- invertedIndexCols.add(INT_COL_NAME);
- invertedIndexCols.add(LOW_CARDINALITY_STRING_COL);
-
- Map<String, BloomFilterConfig> bloomFilterConfigMap = new HashMap<>();
- bloomFilterConfigMap.put(SORTED_COL_NAME, new
BloomFilterConfig(BloomFilterConfig.DEFAULT_FPP, 10000, false));
-
- indexLoadingConfig.setRangeIndexColumns(invertedIndexCols);
- indexLoadingConfig.setInvertedIndexColumns(invertedIndexCols);
- indexLoadingConfig.setBloomFilterConfigs(bloomFilterConfigMap);
+ IndexLoadingConfig indexLoadingConfig = new
IndexLoadingConfig(TABLE_CONFIG, SCHEMA);
_indexSegments = new ArrayList<>();
for (int i = 0; i < _numSegments; i++) {
@@ -178,30 +180,7 @@ public class BenchmarkColumnValueSegmentPruner {
private void buildSegment(String segmentName)
throws Exception {
List<GenericRow> rows = createTestData(_numRows);
- List<FieldConfig> fieldConfigs = new ArrayList<>();
-
- TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
- .setInvertedIndexColumns(Collections.singletonList(INT_COL_NAME))
- .setFieldConfigList(fieldConfigs)
- .setNoDictionaryColumns(Arrays.asList(RAW_INT_COL_NAME,
RAW_STRING_COL_NAME))
- .setSortedColumn(SORTED_COL_NAME)
-
.setVarLengthDictionaryColumns(Collections.singletonList(SORTED_COL_NAME))
- .setBloomFilterColumns(Collections.singletonList(SORTED_COL_NAME))
- .setStarTreeIndexConfigs(Collections.singletonList(
- new StarTreeIndexConfig(Arrays.asList(SORTED_COL_NAME,
INT_COL_NAME), null, Collections.singletonList(
- new AggregationFunctionColumnPair(AggregationFunctionType.SUM,
RAW_INT_COL_NAME).toColumnName()), null,
- Integer.MAX_VALUE)))
- .build();
- Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
- .addSingleValueDimension(SORTED_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(NO_INDEX_INT_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(RAW_INT_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(INT_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(RAW_STRING_COL_NAME,
FieldSpec.DataType.STRING)
- .addSingleValueDimension(NO_INDEX_STRING_COL,
FieldSpec.DataType.STRING)
- .addSingleValueDimension(LOW_CARDINALITY_STRING_COL,
FieldSpec.DataType.STRING)
- .build();
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(TABLE_CONFIG,
SCHEMA);
config.setOutDir(INDEX_DIR.getPath());
config.setTableName(TABLE_NAME);
config.setSegmentName(segmentName);
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkNativeAndLuceneBasedLike.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkNativeAndLuceneBasedLike.java
index 4ed28c746f..548f7d12ba 100644
---
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkNativeAndLuceneBasedLike.java
+++
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkNativeAndLuceneBasedLike.java
@@ -21,9 +21,7 @@ package org.apache.pinot.perf;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.core.common.Operator;
@@ -81,6 +79,9 @@ public class BenchmarkNativeAndLuceneBasedLike {
private static final String URL_COL = "URL_COL";
private static final String INT_COL_NAME = "INT_COL";
private static final String NO_INDEX_STRING_COL_NAME = "NO_INDEX_COL";
+ private TableConfig _tableConfig;
+ private List<FieldConfig> _fieldConfigs = new ArrayList<>();
+ private Schema _schema;
@Param({"LUCENE", "NATIVE"})
private FSTType _fstType;
@@ -105,11 +106,8 @@ public class BenchmarkNativeAndLuceneBasedLike {
_queryContext = QueryContextConverterUtils.getQueryContext(_query);
FileUtils.deleteQuietly(INDEX_DIR);
buildSegment(_fstType);
- IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig();
- Set<String> fstIndexCols = new HashSet<>();
- fstIndexCols.add(DOMAIN_NAMES_COL);
- indexLoadingConfig.setFSTIndexColumns(fstIndexCols);
- indexLoadingConfig.setFSTIndexType(_fstType);
+
+ IndexLoadingConfig indexLoadingConfig = new
IndexLoadingConfig(_tableConfig, _schema);
_indexSegment = ImmutableSegmentLoader.load(new File(INDEX_DIR,
SEGMENT_NAME), indexLoadingConfig);
}
@@ -145,19 +143,18 @@ public class BenchmarkNativeAndLuceneBasedLike {
private void buildSegment(FSTType fstType)
throws Exception {
List<GenericRow> rows = createTestData(_numRows);
- List<FieldConfig> fieldConfigs = new ArrayList<>();
- fieldConfigs.add(
+ _fieldConfigs.add(
new FieldConfig(DOMAIN_NAMES_COL, FieldConfig.EncodingType.DICTIONARY,
FieldConfig.IndexType.FST, null, null));
- fieldConfigs
+ _fieldConfigs
.add(new FieldConfig(URL_COL, FieldConfig.EncodingType.DICTIONARY,
FieldConfig.IndexType.FST, null, null));
- TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
-
.setInvertedIndexColumns(Collections.singletonList(DOMAIN_NAMES_COL)).setFieldConfigList(fieldConfigs).build();
- Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ _tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
+
.setInvertedIndexColumns(Collections.singletonList(DOMAIN_NAMES_COL)).setFieldConfigList(_fieldConfigs).build();
+ _schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
.addSingleValueDimension(DOMAIN_NAMES_COL, FieldSpec.DataType.STRING)
.addSingleValueDimension(URL_COL, FieldSpec.DataType.STRING)
.addSingleValueDimension(NO_INDEX_STRING_COL_NAME,
FieldSpec.DataType.STRING)
.addMetric(INT_COL_NAME, FieldSpec.DataType.INT).build();
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(_tableConfig,
_schema);
config.setOutDir(INDEX_DIR.getPath());
config.setTableName(TABLE_NAME);
config.setSegmentName(SEGMENT_NAME);
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOrderByQueries.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOrderByQueries.java
index 2c72421893..e627550969 100644
---
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOrderByQueries.java
+++
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOrderByQueries.java
@@ -23,10 +23,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.LongSupplier;
@@ -94,6 +92,28 @@ public class BenchmarkOrderByQueries extends BaseQueriesTest
{
private static final String NO_INDEX_INT_COL_NAME = "NO_INDEX_INT_COL";
private static final String NO_INDEX_STRING_COL = "NO_INDEX_STRING_COL";
private static final String LOW_CARDINALITY_STRING_COL =
"LOW_CARDINALITY_STRING_COL";
+ private static final List<FieldConfig> FIELD_CONFIGS = new ArrayList<>();
+
+ private static final TableConfig TABLE_CONFIG = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
+ .setInvertedIndexColumns(List.of(INT_COL_NAME,
LOW_CARDINALITY_STRING_COL))
+ .setFieldConfigList(FIELD_CONFIGS)
+ .setNoDictionaryColumns(List.of(RAW_INT_COL_NAME, RAW_STRING_COL_NAME))
+ .setSortedColumn(SORTED_COL_NAME)
+ .setRangeIndexColumns(List.of(INT_COL_NAME, LOW_CARDINALITY_STRING_COL))
+ .setStarTreeIndexConfigs(Collections.singletonList(
+ new StarTreeIndexConfig(Arrays.asList(SORTED_COL_NAME,
INT_COL_NAME), null, Collections.singletonList(
+ new AggregationFunctionColumnPair(AggregationFunctionType.SUM,
RAW_INT_COL_NAME).toColumnName()), null,
+ Integer.MAX_VALUE)))
+ .build();
+ private static final Schema SCHEMA = new
Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ .addSingleValueDimension(SORTED_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(NO_INDEX_INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(RAW_INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(RAW_STRING_COL_NAME, FieldSpec.DataType.STRING)
+ .addSingleValueDimension(NO_INDEX_STRING_COL, FieldSpec.DataType.STRING)
+ .addSingleValueDimension(LOW_CARDINALITY_STRING_COL,
FieldSpec.DataType.STRING)
+ .build();
@Param("1500000")
private int _numRows;
@@ -115,14 +135,8 @@ public class BenchmarkOrderByQueries extends
BaseQueriesTest {
buildSegment(FIRST_SEGMENT_NAME);
buildSegment(SECOND_SEGMENT_NAME);
- IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig();
- Set<String> invertedIndexCols = new HashSet<>();
- invertedIndexCols.add(INT_COL_NAME);
- invertedIndexCols.add(LOW_CARDINALITY_STRING_COL);
-
- indexLoadingConfig.setRangeIndexColumns(invertedIndexCols);
- indexLoadingConfig.setInvertedIndexColumns(invertedIndexCols);
+ IndexLoadingConfig indexLoadingConfig = new
IndexLoadingConfig(TABLE_CONFIG, SCHEMA);
ImmutableSegment firstImmutableSegment =
ImmutableSegmentLoader.load(new File(INDEX_DIR, FIRST_SEGMENT_NAME),
indexLoadingConfig);
@@ -167,28 +181,7 @@ public class BenchmarkOrderByQueries extends
BaseQueriesTest {
private void buildSegment(String segmentName)
throws Exception {
List<GenericRow> rows = createTestData(_numRows);
- List<FieldConfig> fieldConfigs = new ArrayList<>();
-
- TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
- .setInvertedIndexColumns(Collections.singletonList(INT_COL_NAME))
- .setFieldConfigList(fieldConfigs)
- .setNoDictionaryColumns(Arrays.asList(RAW_INT_COL_NAME,
RAW_STRING_COL_NAME))
- .setSortedColumn(SORTED_COL_NAME)
- .setStarTreeIndexConfigs(Collections.singletonList(
- new StarTreeIndexConfig(Arrays.asList(SORTED_COL_NAME,
INT_COL_NAME), null, Collections.singletonList(
- new AggregationFunctionColumnPair(AggregationFunctionType.SUM,
RAW_INT_COL_NAME).toColumnName()), null,
- Integer.MAX_VALUE)))
- .build();
- Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
- .addSingleValueDimension(SORTED_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(NO_INDEX_INT_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(RAW_INT_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(INT_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(RAW_STRING_COL_NAME,
FieldSpec.DataType.STRING)
- .addSingleValueDimension(NO_INDEX_STRING_COL,
FieldSpec.DataType.STRING)
- .addSingleValueDimension(LOW_CARDINALITY_STRING_COL,
FieldSpec.DataType.STRING)
- .build();
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(TABLE_CONFIG,
SCHEMA);
config.setOutDir(INDEX_DIR.getPath());
config.setTableName(TABLE_NAME);
config.setSegmentName(segmentName);
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueries.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueries.java
index d5002c09b6..3b3a58c1da 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueries.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueries.java
@@ -23,10 +23,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.LongSupplier;
@@ -94,6 +92,25 @@ public class BenchmarkQueries extends BaseQueriesTest {
private static final String NO_INDEX_INT_COL_NAME = "NO_INDEX_INT_COL";
private static final String NO_INDEX_STRING_COL = "NO_INDEX_STRING_COL";
private static final String LOW_CARDINALITY_STRING_COL =
"LOW_CARDINALITY_STRING_COL";
+ private static final List<FieldConfig> FIELD_CONFIGS = new ArrayList<>();
+
+ private static final TableConfig TABLE_CONFIG = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
+ .setInvertedIndexColumns(List.of(INT_COL_NAME,
LOW_CARDINALITY_STRING_COL)).setFieldConfigList(FIELD_CONFIGS)
+ .setNoDictionaryColumns(List.of(RAW_INT_COL_NAME,
RAW_STRING_COL_NAME)).setSortedColumn(SORTED_COL_NAME)
+ .setRangeIndexColumns(List.of(INT_COL_NAME,
LOW_CARDINALITY_STRING_COL)).setStarTreeIndexConfigs(
+ Collections.singletonList(new
StarTreeIndexConfig(List.of(SORTED_COL_NAME, INT_COL_NAME), null,
+ Collections.singletonList(
+ new
AggregationFunctionColumnPair(AggregationFunctionType.SUM,
RAW_INT_COL_NAME).toColumnName()),
+ null, Integer.MAX_VALUE))).build();
+ private static final Schema SCHEMA = new
Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ .addSingleValueDimension(SORTED_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(NO_INDEX_INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(RAW_INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(RAW_STRING_COL_NAME, FieldSpec.DataType.STRING)
+ .addSingleValueDimension(NO_INDEX_STRING_COL, FieldSpec.DataType.STRING)
+ .addSingleValueDimension(LOW_CARDINALITY_STRING_COL,
FieldSpec.DataType.STRING)
+ .build();
public static final String FILTERED_QUERY = "SELECT SUM(INT_COL)
FILTER(WHERE INT_COL > 123 AND INT_COL < 599999),"
+ "MAX(INT_COL) FILTER(WHERE INT_COL > 123 AND INT_COL < 599999) "
@@ -190,15 +207,8 @@ public class BenchmarkQueries extends BaseQueriesTest {
buildSegment(FIRST_SEGMENT_NAME);
buildSegment(SECOND_SEGMENT_NAME);
- IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig();
-
- Set<String> invertedIndexCols = new HashSet<>();
- invertedIndexCols.add(INT_COL_NAME);
- invertedIndexCols.add(LOW_CARDINALITY_STRING_COL);
-
- indexLoadingConfig.setRangeIndexColumns(invertedIndexCols);
- indexLoadingConfig.setInvertedIndexColumns(invertedIndexCols);
+ IndexLoadingConfig indexLoadingConfig = new
IndexLoadingConfig(TABLE_CONFIG, SCHEMA);
ImmutableSegment firstImmutableSegment =
ImmutableSegmentLoader.load(new File(INDEX_DIR, FIRST_SEGMENT_NAME),
indexLoadingConfig);
ImmutableSegment secondImmutableSegment =
@@ -240,28 +250,7 @@ public class BenchmarkQueries extends BaseQueriesTest {
private void buildSegment(String segmentName)
throws Exception {
List<GenericRow> rows = createTestData(_numRows);
- List<FieldConfig> fieldConfigs = new ArrayList<>();
-
- TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
- .setInvertedIndexColumns(Collections.singletonList(INT_COL_NAME))
- .setFieldConfigList(fieldConfigs)
- .setNoDictionaryColumns(Arrays.asList(RAW_INT_COL_NAME,
RAW_STRING_COL_NAME))
- .setSortedColumn(SORTED_COL_NAME)
- .setStarTreeIndexConfigs(Collections.singletonList(
- new StarTreeIndexConfig(Arrays.asList(SORTED_COL_NAME,
INT_COL_NAME), null, Collections.singletonList(
- new AggregationFunctionColumnPair(AggregationFunctionType.SUM,
RAW_INT_COL_NAME).toColumnName()), null,
- Integer.MAX_VALUE)))
- .build();
- Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
- .addSingleValueDimension(SORTED_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(NO_INDEX_INT_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(RAW_INT_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(INT_COL_NAME, FieldSpec.DataType.INT)
- .addSingleValueDimension(RAW_STRING_COL_NAME,
FieldSpec.DataType.STRING)
- .addSingleValueDimension(NO_INDEX_STRING_COL,
FieldSpec.DataType.STRING)
- .addSingleValueDimension(LOW_CARDINALITY_STRING_COL,
FieldSpec.DataType.STRING)
- .build();
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(TABLE_CONFIG,
SCHEMA);
config.setOutDir(INDEX_DIR.getPath());
config.setTableName(TABLE_NAME);
config.setSegmentName(segmentName);
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/StringDictionaryPerfTest.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/StringDictionaryPerfTest.java
index 0344e4827f..9594e2a534 100644
---
a/pinot-perf/src/main/java/org/apache/pinot/perf/StringDictionaryPerfTest.java
+++
b/pinot-perf/src/main/java/org/apache/pinot/perf/StringDictionaryPerfTest.java
@@ -60,11 +60,15 @@ public class StringDictionaryPerfTest {
"Kurtosis", "Variance", "BufferSize"
};
private static final Joiner COMMA_JOINER = Joiner.on(",");
+ private static final TableConfig TABLE_CONFIG =
+ new
TableConfigBuilder(TableType.OFFLINE).setOnHeapDictionaryColumns(List.of(COLUMN_NAME)).setTableName("test")
+ .build();
private final DescriptiveStatistics _statistics = new
DescriptiveStatistics();
private String[] _inputStrings;
private File _indexDir;
private int _dictLength;
+ private Schema _schema;
/**
* Helper method to build a segment:
@@ -75,19 +79,18 @@ public class StringDictionaryPerfTest {
*/
private void buildSegment(int dictLength)
throws Exception {
- Schema schema = new Schema();
+ _schema = new Schema();
String segmentName = "perfTestSegment" + System.currentTimeMillis();
_indexDir = new File(TMP_DIR + File.separator + segmentName);
_indexDir.deleteOnExit();
FieldSpec fieldSpec = new DimensionFieldSpec(COLUMN_NAME,
FieldSpec.DataType.STRING, true);
- schema.addField(fieldSpec);
- TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
+ _schema.addField(fieldSpec);
_dictLength = dictLength;
_inputStrings = new String[dictLength];
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(TABLE_CONFIG,
_schema);
config.setOutDir(_indexDir.getParent());
config.setFormat(FileFormat.AVRO);
config.setSegmentName(segmentName);
@@ -151,11 +154,7 @@ public class StringDictionaryPerfTest {
Runtime r = Runtime.getRuntime();
System.gc();
long oldMemory = r.totalMemory() - r.freeMemory();
- IndexLoadingConfig defaultIndexLoadingConfig = new IndexLoadingConfig();
- defaultIndexLoadingConfig.setReadMode(ReadMode.heap);
- Set<String> columnNames = new HashSet<>();
- columnNames.add(COLUMN_NAME);
- defaultIndexLoadingConfig.setOnHeapDictionaryColumns(columnNames);
+ IndexLoadingConfig defaultIndexLoadingConfig = new
IndexLoadingConfig(TABLE_CONFIG, _schema);
ImmutableSegment immutableSegment = ImmutableSegmentLoader.load(_indexDir,
defaultIndexLoadingConfig);
Dictionary dictionary = immutableSegment.getDictionary(COLUMN_NAME);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
