This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new d0a87b5 Pass valid tableConfig to SegmentGeneratorConfig in all tests
(#5353)
d0a87b5 is described below
commit d0a87b5d3868d0d1f0427c1ae1f69fad19e511d7
Author: Neha Pawar <[email protected]>
AuthorDate: Fri May 8 17:32:24 2020 -0700
Pass valid tableConfig to SegmentGeneratorConfig in all tests (#5353)
---
.../indexsegment/generator/SegmentGeneratorConfig.java | 16 +++++++---------
.../org/apache/pinot/core/minion/SegmentConverter.java | 8 ++++----
.../org/apache/pinot/core/common/DataFetcherTest.java | 8 ++++++--
.../generator/SegmentGeneratorConfigTest.java | 12 ++++++++++--
.../core/segment/index/creator/RawIndexCreatorTest.java | 11 ++++++++---
.../creator/SegmentGenerationWithBytesTypeTest.java | 12 +++++++++---
.../SegmentGenerationWithNullValueVectorTest.java | 10 +++++++---
.../core/segment/index/creator/SegmentPartitionTest.java | 6 +++++-
.../segment/index/loader/SegmentPreProcessorTest.java | 8 +++++++-
.../pinot/core/startree/v2/BaseStarTreeV2Test.java | 6 +++++-
.../org/apache/pinot/queries/DistinctQueriesTest.java | 5 ++++-
.../org/apache/pinot/queries/FastHllQueriesTest.java | 7 ++++++-
.../pinot/queries/PercentileTDigestMVQueriesTest.java | 6 +++++-
.../pinot/queries/PercentileTDigestQueriesTest.java | 6 +++++-
.../RangePredicateWithSortedInvertedIndexTest.java | 12 +++++++++---
.../apache/pinot/queries/SerializedBytesQueriesTest.java | 7 ++++++-
.../org/apache/pinot/queries/TextSearchQueriesTest.java | 7 ++++++-
.../aggregation/DefaultAggregationExecutorTest.java | 9 +++++----
.../groupby/DictionaryBasedGroupKeyGeneratorTest.java | 7 ++++++-
.../groupby/NoDictionaryGroupKeyGeneratorTest.java | 7 ++++++-
.../segments/v1/creator/OnHeapDictionariesTest.java | 11 ++++++++---
.../pinot/segments/v1/creator/SegmentTestUtils.java | 13 +++++++++----
.../tests/DefaultCommitterRealtimeIntegrationTest.java | 4 +++-
.../org/apache/pinot/server/util/SegmentTestUtils.java | 15 +++++++++++----
.../java/org/apache/pinot/perf/RawIndexBenchmark.java | 7 +++++--
.../org/apache/pinot/perf/StringDictionaryPerfTest.java | 6 +++++-
26 files changed, 167 insertions(+), 59 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java
b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java
index 11facac..252f648 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java
@@ -112,24 +112,22 @@ public class SegmentGeneratorConfig {
* However, for maintaining backward compatibility, taking it
from schema if table config is null.
* This will not work once we start supporting multiple time
columns (DateTimeFieldSpec)
*/
- public SegmentGeneratorConfig(@Nullable TableConfig tableConfig, Schema
schema) {
+ public SegmentGeneratorConfig(TableConfig tableConfig, Schema schema) {
Preconditions.checkNotNull(schema);
+ Preconditions.checkNotNull(tableConfig);
setSchema(schema);
// NOTE: SegmentGeneratorConfig#setSchema doesn't set the time column
anymore. timeColumnName is expected to be read from table config.
- // But table config is not mandatory, and cannot be easily enforced as
the instantiation can happen in external code.
- // Hence, if table config is null, but timeFieldSpec is not null, read
time from schema
- // Once we move to multiple time columns - DateTimeFieldSpec - table
config has to be provided with valid time
- // If more than 1 dateTimeFieldSpec is found along with null table
config, throw exception.
+ // If time column name is not set in table config, read time from schema.
+ // WARN: Once we move to DateTimeFieldSpec - table config has to be
provided with valid time - if time needs to be set.
+ // We cannot deduce whether 1) one of the provided DateTimes should be
used as time column 2) if yes, which one
+ // Even if only 1 DateTime exists, we cannot determine whether it should
be primary time column (there could be no time column for table (REFRESH), but
still multiple DateTimeFieldSpec)
String timeColumnName = null;
- if (tableConfig != null && tableConfig.getValidationConfig() != null) {
+ if (tableConfig.getValidationConfig() != null) {
timeColumnName = tableConfig.getValidationConfig().getTimeColumnName();
}
setTime(timeColumnName, schema);
- if (tableConfig == null) {
- return;
- }
IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
if (indexingConfig != null) {
List<String> noDictionaryColumns =
indexingConfig.getNoDictionaryColumns();
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentConverter.java
b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentConverter.java
index 988891a..c399505 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentConverter.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentConverter.java
@@ -104,7 +104,7 @@ public class SegmentConverter {
try (MapperRecordReader mapperRecordReader = new
MapperRecordReader(_inputIndexDirs, _recordTransformer,
_recordPartitioner, _totalNumPartition, currentPartition)) {
- buildSegment(mapperOutputPath, _tableName, outputSegmentName,
mapperRecordReader,
+ buildSegment(mapperOutputPath, outputSegmentName, mapperRecordReader,
mapperRecordReader.getSchema(), _tableConfig);
}
File outputSegment = new File(mapperOutputPath + File.separator +
outputSegmentName);
@@ -114,7 +114,7 @@ public class SegmentConverter {
String reducerOutputPath = _workingDir.getPath() + File.separator +
REDUCER_PREFIX + currentPartition;
try (ReducerRecordReader reducerRecordReader = new
ReducerRecordReader(outputSegment, _recordAggregator,
_groupByColumns)) {
- buildSegment(reducerOutputPath, _tableName, outputSegmentName,
reducerRecordReader,
+ buildSegment(reducerOutputPath, outputSegmentName,
reducerRecordReader,
reducerRecordReader.getSchema(), _tableConfig);
}
outputSegment = new File(reducerOutputPath + File.separator +
outputSegmentName);
@@ -130,7 +130,7 @@ public class SegmentConverter {
String indexGenerationOutputPath = _workingDir.getPath() +
File.separator + INDEX_PREFIX + currentPartition;
try (PinotSegmentRecordReader pinotSegmentRecordReader = new
PinotSegmentRecordReader(outputSegment, null,
sortedColumn)) {
- buildSegment(indexGenerationOutputPath, _tableName,
outputSegmentName, pinotSegmentRecordReader,
+ buildSegment(indexGenerationOutputPath, outputSegmentName,
pinotSegmentRecordReader,
pinotSegmentRecordReader.getSchema(), _tableConfig);
}
outputSegment = new File(indexGenerationOutputPath + File.separator
+ outputSegmentName);
@@ -147,7 +147,7 @@ public class SegmentConverter {
*
* TODO: Support all kinds of indexing (no dictionary)
*/
- private void buildSegment(String outputPath, String tableName, String
segmentName, RecordReader recordReader,
+ private void buildSegment(String outputPath, String segmentName,
RecordReader recordReader,
Schema schema, TableConfig tableConfig)
throws Exception {
SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(tableConfig, schema);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/common/DataFetcherTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/common/DataFetcherTest.java
index da634cb..7928fb9 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/common/DataFetcherTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/common/DataFetcherTest.java
@@ -26,6 +26,8 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.MetricFieldSpec;
@@ -37,6 +39,7 @@ import org.apache.pinot.core.indexsegment.IndexSegment;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -110,9 +113,10 @@ public class DataFetcherTest {
schema.addField(new MetricFieldSpec(NO_DICT_FLOAT_METRIC_NAME,
FieldSpec.DataType.FLOAT));
schema.addField(new MetricFieldSpec(NO_DICT_DOUBLE_METRIC_NAME,
FieldSpec.DataType.DOUBLE));
- SegmentGeneratorConfig config = new SegmentGeneratorConfig();
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
+
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
FileUtils.deleteQuietly(new File(INDEX_DIR_PATH));
- config.setSchema(schema);
config.setOutDir(INDEX_DIR_PATH);
config.setSegmentName(SEGMENT_NAME);
config.setRawIndexCreationColumns(Arrays
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfigTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfigTest.java
index e217ba0..6c19432 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfigTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfigTest.java
@@ -48,7 +48,11 @@ public class SegmentGeneratorConfigTest {
assertNull(segmentGeneratorConfig.getSimpleDateFormat());
// table config not provided
- segmentGeneratorConfig = new SegmentGeneratorConfig(null, schema);
+ // NOTE: this behavior will not hold true when we move to
dateTimeFieldSpec.
+ // MUST provide valid tableConfig with time column if time details are
wanted
+ tableConfig =
+ new TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
+ segmentGeneratorConfig = new SegmentGeneratorConfig(tableConfig, schema);
assertEquals(segmentGeneratorConfig.getTimeColumnName(), "daysSinceEpoch");
assertEquals(segmentGeneratorConfig.getTimeColumnType(),
SegmentGeneratorConfig.TimeColumnType.EPOCH);
assertEquals(segmentGeneratorConfig.getSegmentTimeUnit(), TimeUnit.DAYS);
@@ -70,7 +74,11 @@ public class SegmentGeneratorConfigTest {
assertEquals(segmentGeneratorConfig.getSimpleDateFormat(), "yyyyMMdd");
// Table config not provided
- segmentGeneratorConfig = new SegmentGeneratorConfig(null, schema);
+ // NOTE: this behavior will not hold true when we move to
dateTimeFieldSpec.
+ // MUST provide valid tableConfig with time column if time details are
wanted
+ tableConfig =
+ new TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
+ segmentGeneratorConfig = new SegmentGeneratorConfig(tableConfig, schema);
assertEquals(segmentGeneratorConfig.getTimeColumnName(), "Date");
assertEquals(segmentGeneratorConfig.getTimeColumnType(),
SegmentGeneratorConfig.TimeColumnType.SIMPLE_DATE);
assertNull(segmentGeneratorConfig.getSegmentTimeUnit());
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/RawIndexCreatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/RawIndexCreatorTest.java
index 7ce7ef5..ad097f9 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/RawIndexCreatorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/RawIndexCreatorTest.java
@@ -26,6 +26,8 @@ import java.util.List;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
@@ -42,6 +44,7 @@ import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl
import org.apache.pinot.core.segment.memory.PinotDataBuffer;
import org.apache.pinot.core.segment.store.ColumnIndexType;
import org.apache.pinot.core.segment.store.SegmentDirectory;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -84,8 +87,10 @@ public class RawIndexCreatorTest {
schema.addField(new DimensionFieldSpec(DOUBLE_COLUMN,
FieldSpec.DataType.DOUBLE, true));
schema.addField(new DimensionFieldSpec(STRING_COLUMN,
FieldSpec.DataType.STRING, true));
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
+
_random = new Random(System.nanoTime());
- _recordReader = buildIndex(schema);
+ _recordReader = buildIndex(tableConfig, schema);
}
/**
@@ -202,9 +207,9 @@ public class RawIndexCreatorTest {
* @return Array of string values for the rows in the generated index.
* @throws Exception
*/
- private RecordReader buildIndex(Schema schema)
+ private RecordReader buildIndex(TableConfig tableConfig, Schema schema)
throws Exception {
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(null, schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
config.setRawIndexCreationColumns(schema.getDimensionNames());
config.setOutDir(SEGMENT_DIR_NAME);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
index dcc83b8..efff01e 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
@@ -45,6 +45,8 @@ import
org.apache.pinot.core.segment.index.metadata.SegmentMetadata;
import org.apache.pinot.core.segment.index.readers.BaseImmutableDictionary;
import org.apache.pinot.core.segment.store.SegmentDirectory;
import org.apache.pinot.plugin.inputformat.avro.AvroUtils;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.MetricFieldSpec;
@@ -53,6 +55,7 @@ import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.RecordReader;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.BytesUtils;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -84,6 +87,7 @@ public class SegmentGenerationWithBytesTypeTest {
private Random _random;
private RecordReader _recordReader;
private Schema _schema;
+ private TableConfig _tableConfig;
private ImmutableSegment _segment;
/**
@@ -101,6 +105,8 @@ public class SegmentGenerationWithBytesTypeTest {
_schema.addField(new DimensionFieldSpec(FIXED_BYTES_NO_DICT_COLUMN,
FieldSpec.DataType.BYTES, true));
_schema.addField(new DimensionFieldSpec(VARIABLE_BYTES_COLUMN,
FieldSpec.DataType.BYTES, true));
+ _tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
+
_random = new Random(System.nanoTime());
_recordReader = buildIndex(_schema);
_segment = ImmutableSegmentLoader.load(new File(SEGMENT_DIR_NAME,
SEGMENT_NAME), ReadMode.heap);
@@ -139,7 +145,7 @@ public class SegmentGenerationWithBytesTypeTest {
}
// Ensure both record readers are exhausted, ie same number of rows.
- Assert.assertTrue(!_recordReader.hasNext());
+ Assert.assertFalse(_recordReader.hasNext());
pinotReader.close();
}
@@ -223,7 +229,7 @@ public class SegmentGenerationWithBytesTypeTest {
private RecordReader buildIndex(Schema schema)
throws Exception {
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(null, schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(_tableConfig,
schema);
config.setOutDir(SEGMENT_DIR_NAME);
config.setSegmentName(SEGMENT_NAME);
@@ -321,7 +327,7 @@ public class SegmentGenerationWithBytesTypeTest {
*/
private IndexSegment buildSegmentFromAvro(Schema schema, String dirName,
String avroName, String segmentName)
throws Exception {
- SegmentGeneratorConfig config = new SegmentGeneratorConfig();
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(_tableConfig,
schema);
config.setInputFilePath(dirName + File.separator + avroName);
config.setOutDir(dirName);
config.setSegmentName(segmentName);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/SegmentGenerationWithNullValueVectorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/SegmentGenerationWithNullValueVectorTest.java
index f568d7b..4d8402f 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/SegmentGenerationWithNullValueVectorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/SegmentGenerationWithNullValueVectorTest.java
@@ -46,6 +46,8 @@ import org.apache.pinot.core.query.executor.QueryExecutor;
import org.apache.pinot.core.query.executor.ServerQueryExecutorV1Impl;
import org.apache.pinot.core.query.request.ServerQueryRequest;
import org.apache.pinot.pql.parsers.Pql2Compiler;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
@@ -58,6 +60,7 @@ import
org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
import org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.core.segment.index.readers.NullValueVectorReader;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -119,8 +122,9 @@ public class SegmentGenerationWithNullValueVectorTest {
_schema.addField(new DimensionFieldSpec(DOUBLE_COLUMN,
FieldSpec.DataType.DOUBLE, true));
_schema.addField(new DimensionFieldSpec(STRING_COLUMN,
FieldSpec.DataType.STRING, true));
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
_random = new Random(System.nanoTime());
- buildIndex(_schema);
+ buildIndex(tableConfig, _schema);
_segment = ImmutableSegmentLoader.load(new File(SEGMENT_DIR_NAME,
SEGMENT_NAME), ReadMode.heap);
setupQueryServer();
@@ -162,9 +166,9 @@ public class SegmentGenerationWithNullValueVectorTest {
* @throws Exception
*/
- private void buildIndex(Schema schema)
+ private void buildIndex(TableConfig tableConfig, Schema schema)
throws Exception {
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(null, schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
config.setOutDir(SEGMENT_DIR_NAME);
config.setSegmentName(SEGMENT_NAME);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/SegmentPartitionTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/SegmentPartitionTest.java
index e34aaf6..c670317 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/SegmentPartitionTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/SegmentPartitionTest.java
@@ -41,11 +41,14 @@ import
org.apache.pinot.core.segment.index.metadata.ColumnMetadata;
import org.apache.pinot.core.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.SegmentPartitionConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -189,6 +192,7 @@ public class SegmentPartitionTest {
Schema schema = new Schema();
schema.addField(new DimensionFieldSpec(PARTITIONED_COLUMN_NAME,
FieldSpec.DataType.INT, true));
schema.addField(new DimensionFieldSpec(NON_PARTITIONED_COLUMN_NAME,
FieldSpec.DataType.INT, true));
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
Random random = new Random();
Map<String, ColumnPartitionConfig> partitionFunctionMap = new HashMap<>();
@@ -197,7 +201,7 @@ public class SegmentPartitionTest {
.put(PARTITIONED_COLUMN_NAME, new
ColumnPartitionConfig(PARTITION_FUNCTION_NAME, NUM_PARTITIONS));
SegmentPartitionConfig segmentPartitionConfig = new
SegmentPartitionConfig(partitionFunctionMap);
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(null, schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
config.setOutDir(SEGMENT_DIR_NAME);
config.setSegmentName(SEGMENT_NAME);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/segment/index/loader/SegmentPreProcessorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/segment/index/loader/SegmentPreProcessorTest.java
index a2a2788..edc9fbe 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/segment/index/loader/SegmentPreProcessorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/segment/index/loader/SegmentPreProcessorTest.java
@@ -42,8 +42,11 @@ import org.apache.pinot.core.segment.store.ColumnIndexType;
import org.apache.pinot.core.segment.store.SegmentDirectory;
import org.apache.pinot.core.segment.store.SegmentDirectoryPaths;
import org.apache.pinot.segments.v1.creator.SegmentTestUtils;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -87,6 +90,7 @@ public class SegmentPreProcessorTest {
private IndexLoadingConfig _indexLoadingConfig;
private File _avroFile;
private Schema _schema;
+ private TableConfig _tableConfig;
private Schema _newColumnsSchema1;
private Schema _newColumnsSchema2;
private Schema _newColumnsSchema3;
@@ -113,6 +117,8 @@ public class SegmentPreProcessorTest {
resourceUrl = classLoader.getResource(SCHEMA);
Assert.assertNotNull(resourceUrl);
_schema = Schema.fromFile(new File(resourceUrl.getFile()));
+ _tableConfig =
+ new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").setTimeColumnName("daySinceEpoch").build();
resourceUrl = classLoader.getResource(NEW_COLUMNS_SCHEMA1);
Assert.assertNotNull(resourceUrl);
_newColumnsSchema1 = Schema.fromFile(new File(resourceUrl.getFile()));
@@ -133,7 +139,7 @@ public class SegmentPreProcessorTest {
// Create inverted index for 'column7' when constructing the segment.
SegmentGeneratorConfig segmentGeneratorConfig =
- SegmentTestUtils.getSegmentGeneratorConfigWithSchema(_avroFile,
INDEX_DIR, "testTable", _schema);
+ SegmentTestUtils.getSegmentGeneratorConfigWithSchema(_avroFile,
INDEX_DIR, "testTable", _tableConfig, _schema);
segmentGeneratorConfig.setInvertedIndexCreationColumns(Collections.singletonList(COLUMN7_NAME));
segmentGeneratorConfig.setRawIndexCreationColumns(Collections.singletonList(EXISTING_STRING_COL_RAW));
// The segment generation code in SegmentColumnarIndexCreator will throw
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
index bd6b568..03177ae 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
@@ -30,6 +30,8 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.common.function.AggregationFunctionType;
@@ -60,6 +62,7 @@ import
org.apache.pinot.core.startree.v2.builder.MultipleTreesBuilder;
import
org.apache.pinot.core.startree.v2.builder.MultipleTreesBuilder.BuildMode;
import org.apache.pinot.core.startree.v2.builder.StarTreeV2BuilderConfig;
import org.apache.pinot.pql.parsers.Pql2Compiler;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -111,6 +114,7 @@ abstract class BaseStarTreeV2Test<R, A> {
schemaBuilder.addMetric(METRIC, rawValueType);
}
Schema schema = schemaBuilder.build();
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
List<GenericRow> segmentRecords = new ArrayList<>(NUM_SEGMENT_RECORDS);
for (int i = 0; i < NUM_SEGMENT_RECORDS; i++) {
@@ -126,7 +130,7 @@ abstract class BaseStarTreeV2Test<R, A> {
}
SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
- SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(null, schema);
+ SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(tableConfig, schema);
segmentGeneratorConfig.setOutDir(TEMP_DIR.getPath());
segmentGeneratorConfig.setSegmentName(SEGMENT_NAME);
driver.init(segmentGeneratorConfig, new
GenericRowRecordReader(segmentRecords));
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/DistinctQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/DistinctQueriesTest.java
index 28f646b..6417788 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/DistinctQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/DistinctQueriesTest.java
@@ -53,12 +53,14 @@ import
org.apache.pinot.core.query.aggregation.DistinctTable;
import org.apache.pinot.core.query.reduce.BrokerReduceService;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.core.transport.ServerRoutingInstance;
+import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.BytesUtils;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -89,6 +91,7 @@ public class DistinctQueriesTest extends BaseQueriesTest {
.addSingleValueDimension(LONG_COLUMN,
DataType.LONG).addSingleValueDimension(FLOAT_COLUMN, DataType.FLOAT)
.addSingleValueDimension(DOUBLE_COLUMN,
DataType.DOUBLE).addSingleValueDimension(STRING_COLUMN, DataType.STRING)
.addSingleValueDimension(BYTES_COLUMN, DataType.BYTES).build();
+ private static final TableConfig TABLE = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
private IndexSegment _indexSegment;
private List<SegmentDataManager> _segmentDataManagers;
@@ -151,7 +154,7 @@ public class DistinctQueriesTest extends BaseQueriesTest {
throws Exception {
String segmentName = SEGMENT_NAME_PREFIX + index;
- SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(null, SCHEMA);
+ SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(TABLE, SCHEMA);
segmentGeneratorConfig.setTableName(RAW_TABLE_NAME);
segmentGeneratorConfig.setSegmentName(segmentName);
segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath());
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/FastHllQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/FastHllQueriesTest.java
index 8b95d9b..41f7b9e 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/FastHllQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/FastHllQueriesTest.java
@@ -41,9 +41,12 @@ import
org.apache.pinot.core.query.aggregation.groupby.AggregationGroupByResult;
import org.apache.pinot.core.query.aggregation.groupby.GroupKeyGenerator;
import org.apache.pinot.core.segment.creator.SegmentIndexCreationDriver;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.TimeGranularitySpec;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -169,9 +172,11 @@ public class FastHllQueriesTest extends BaseQueriesTest {
.addTime(new TimeGranularitySpec(FieldSpec.DataType.INT,
TimeUnit.DAYS, "daysSinceEpoch"), null)
.addSingleValueDimension("column17_HLL", FieldSpec.DataType.STRING)
.addSingleValueDimension("column18_HLL",
FieldSpec.DataType.STRING).build();
+ TableConfig tableConfig =
+ new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").setTimeColumnName("daysSinceEpoch").build();
// Create the segment generator config
- SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(null, schema);
+ SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(tableConfig, schema);
segmentGeneratorConfig.setInputFilePath(filePath);
segmentGeneratorConfig.setTableName("testTable");
segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath());
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/PercentileTDigestMVQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/PercentileTDigestMVQueriesTest.java
index ade08cf..254adaa 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/PercentileTDigestMVQueriesTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/PercentileTDigestMVQueriesTest.java
@@ -24,6 +24,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.MetricFieldSpec;
@@ -34,6 +36,7 @@ import org.apache.pinot.spi.data.readers.RecordReader;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import
org.apache.pinot.core.query.aggregation.function.PercentileTDigestAggregationFunction;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
/**
@@ -83,8 +86,9 @@ public class PercentileTDigestMVQueriesTest extends
PercentileTDigestQueriesTest
schema.addField(new DimensionFieldSpec(DOUBLE_COLUMN,
FieldSpec.DataType.DOUBLE, false));
schema.addField(new MetricFieldSpec(TDIGEST_COLUMN,
FieldSpec.DataType.BYTES));
schema.addField(new DimensionFieldSpec(GROUP_BY_COLUMN,
FieldSpec.DataType.STRING, true));
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(null, schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
config.setOutDir(INDEX_DIR.getPath());
config.setTableName(TABLE_NAME);
config.setSegmentName(SEGMENT_NAME);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/PercentileTDigestQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/PercentileTDigestQueriesTest.java
index a0d8c6b..dcae9e7 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/PercentileTDigestQueriesTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/PercentileTDigestQueriesTest.java
@@ -31,6 +31,8 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.MetricFieldSpec;
@@ -55,6 +57,7 @@ import
org.apache.pinot.core.query.aggregation.function.PercentileTDigestAggrega
import
org.apache.pinot.core.query.aggregation.groupby.AggregationGroupByResult;
import org.apache.pinot.core.query.aggregation.groupby.GroupKeyGenerator;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -145,8 +148,9 @@ public class PercentileTDigestQueriesTest extends
BaseQueriesTest {
schema.addField(new MetricFieldSpec(DOUBLE_COLUMN,
FieldSpec.DataType.DOUBLE));
schema.addField(new MetricFieldSpec(TDIGEST_COLUMN,
FieldSpec.DataType.BYTES));
schema.addField(new DimensionFieldSpec(GROUP_BY_COLUMN,
FieldSpec.DataType.STRING, true));
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(null, schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
config.setOutDir(INDEX_DIR.getPath());
config.setTableName(TABLE_NAME);
config.setSegmentName(SEGMENT_NAME);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/RangePredicateWithSortedInvertedIndexTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/RangePredicateWithSortedInvertedIndexTest.java
index 2aa55cf..2f34a01 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/RangePredicateWithSortedInvertedIndexTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/RangePredicateWithSortedInvertedIndexTest.java
@@ -36,10 +36,13 @@ import
org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
import org.apache.pinot.core.operator.blocks.IntermediateResultsBlock;
import org.apache.pinot.core.operator.query.SelectionOnlyOperator;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.RecordReader;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -68,6 +71,7 @@ public class RangePredicateWithSortedInvertedIndexTest
extends BaseQueriesTest {
private final long[] longValues = new long[NUM_ROWS];
private Schema _schema;
+ private TableConfig _tableConfig;
@BeforeClass
public void setUp() {
@@ -84,6 +88,7 @@ public class RangePredicateWithSortedInvertedIndexTest
extends BaseQueriesTest {
_schema =
new
Schema.SchemaBuilder().setSchemaName(TABLE_NAME).addSingleValueDimension(D1,
FieldSpec.DataType.STRING)
.addMetric(M1, FieldSpec.DataType.INT).addMetric(M2,
FieldSpec.DataType.LONG).build();
+ _tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
}
private void createTestData() {
@@ -114,9 +119,10 @@ public class RangePredicateWithSortedInvertedIndexTest
extends BaseQueriesTest {
return null;
}
- private void createSegment(Schema schema, RecordReader recordReader, String
segmentName, String tableName)
+ private void createSegment(TableConfig tableConfig, Schema schema,
RecordReader recordReader, String segmentName,
+ String tableName)
throws Exception {
- SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(null, schema);
+ SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(tableConfig, schema);
segmentGeneratorConfig.setTableName(tableName);
segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath());
segmentGeneratorConfig.setSegmentName(segmentName);
@@ -141,7 +147,7 @@ public class RangePredicateWithSortedInvertedIndexTest
extends BaseQueriesTest {
throws Exception {
Random random = new Random();
try (RecordReader recordReader = new GenericRowRecordReader(_rows)) {
- createSegment(_schema, recordReader, SEGMENT_NAME_1, TABLE_NAME);
+ createSegment(_tableConfig, _schema, recordReader, SEGMENT_NAME_1,
TABLE_NAME);
final ImmutableSegment immutableSegment = loadSegment(SEGMENT_NAME_1);
_indexSegments.add(immutableSegment);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/SerializedBytesQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/SerializedBytesQueriesTest.java
index 889695e..29300ad 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/SerializedBytesQueriesTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/SerializedBytesQueriesTest.java
@@ -29,6 +29,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Random;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.common.response.broker.AggregationResult;
@@ -54,6 +56,7 @@ import
org.apache.pinot.core.query.aggregation.function.customobject.QuantileDig
import
org.apache.pinot.core.query.aggregation.groupby.AggregationGroupByResult;
import org.apache.pinot.core.query.aggregation.groupby.GroupKeyGenerator;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -207,7 +210,9 @@ public class SerializedBytesQueriesTest extends
BaseQueriesTest {
.addMetric(DISTINCT_COUNT_HLL_COLUMN,
DataType.BYTES).addMetric(MIN_MAX_RANGE_COLUMN, DataType.BYTES)
.addMetric(PERCENTILE_EST_COLUMN,
DataType.BYTES).addMetric(PERCENTILE_TDIGEST_COLUMN, DataType.BYTES).build();
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(null, schema);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
config.setOutDir(INDEX_DIR.getPath());
config.setTableName(RAW_TABLE_NAME);
config.setSegmentName(SEGMENT_NAME);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesTest.java
index fa0686c..da29a42 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesTest.java
@@ -63,10 +63,13 @@ import
org.apache.pinot.core.operator.query.AggregationOperator;
import org.apache.pinot.core.operator.query.SelectionOnlyOperator;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.RecordReader;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -94,6 +97,7 @@ public class TextSearchQueriesTest extends BaseQueriesTest {
private List<GenericRow> _rows = new ArrayList<>();
private RecordReader _recordReader;
Schema _schema;
+ private TableConfig _tableConfig;
private List<IndexSegment> _indexSegments = new ArrayList<>(1);
private List<SegmentDataManager> _segmentDataManagers = new ArrayList<>();
@@ -139,13 +143,14 @@ public class TextSearchQueriesTest extends
BaseQueriesTest {
.addSingleValueDimension(QUERY_LOG_TEXT_COL_NAME,
FieldSpec.DataType.STRING)
.addSingleValueDimension(SKILLS_TEXT_COL_NAME,
FieldSpec.DataType.STRING)
.addMetric(INT_COL_NAME, FieldSpec.DataType.INT).build();
+ _tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
}
private void createSegment()
throws Exception {
textIndexColumns.add(QUERY_LOG_TEXT_COL_NAME);
textIndexColumns.add(SKILLS_TEXT_COL_NAME);
- SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(null, _schema);
+ SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(_tableConfig, _schema);
segmentGeneratorConfig.setTableName(TABLE_NAME);
segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath());
segmentGeneratorConfig.setSegmentName(SEGMENT_NAME);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/query/aggregation/DefaultAggregationExecutorTest.java
b/pinot-core/src/test/java/org/apache/pinot/query/aggregation/DefaultAggregationExecutorTest.java
index 5fd7cdc..ee13450 100644
---
a/pinot-core/src/test/java/org/apache/pinot/query/aggregation/DefaultAggregationExecutorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/query/aggregation/DefaultAggregationExecutorTest.java
@@ -47,10 +47,12 @@ import
org.apache.pinot.core.query.aggregation.AggregationFunctionContext;
import org.apache.pinot.core.query.aggregation.DefaultAggregationExecutor;
import
org.apache.pinot.core.query.aggregation.function.AggregationFunctionUtils;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.MetricFieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -169,13 +171,12 @@ public class DefaultAggregationExecutorTest {
FileUtils.deleteQuietly(INDEX_DIR);
}
- SegmentGeneratorConfig config = new SegmentGeneratorConfig();
+ SegmentGeneratorConfig config =
+ new SegmentGeneratorConfig(new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build(),
+ buildSchema());
config.setSegmentName(SEGMENT_NAME);
config.setOutDir(INDEX_DIR.getAbsolutePath());
- Schema schema = buildSchema();
- config.setSchema(schema);
-
List<GenericRow> rows = new ArrayList<>(NUM_ROWS);
for (int i = 0; i < NUM_ROWS; i++) {
Map<String, Object> map = new HashMap<>();
diff --git
a/pinot-core/src/test/java/org/apache/pinot/query/aggregation/groupby/DictionaryBasedGroupKeyGeneratorTest.java
b/pinot-core/src/test/java/org/apache/pinot/query/aggregation/groupby/DictionaryBasedGroupKeyGeneratorTest.java
index 64c8046..2223c32 100644
---
a/pinot-core/src/test/java/org/apache/pinot/query/aggregation/groupby/DictionaryBasedGroupKeyGeneratorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/query/aggregation/groupby/DictionaryBasedGroupKeyGeneratorTest.java
@@ -45,10 +45,13 @@ import
org.apache.pinot.core.query.aggregation.groupby.DictionaryBasedGroupKeyGe
import org.apache.pinot.core.query.aggregation.groupby.GroupKeyGenerator;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.pql.parsers.Pql2Compiler;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -122,7 +125,9 @@ public class DictionaryBasedGroupKeyGeneratorTest {
schema.addField(new DimensionFieldSpec(multiValueColumn,
FieldSpec.DataType.INT, false));
}
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(null, schema);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
+
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
config.setOutDir(INDEX_DIR_PATH);
config.setSegmentName(SEGMENT_NAME);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/query/aggregation/groupby/NoDictionaryGroupKeyGeneratorTest.java
b/pinot-core/src/test/java/org/apache/pinot/query/aggregation/groupby/NoDictionaryGroupKeyGeneratorTest.java
index 9f67005..e8ca22a 100644
---
a/pinot-core/src/test/java/org/apache/pinot/query/aggregation/groupby/NoDictionaryGroupKeyGeneratorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/query/aggregation/groupby/NoDictionaryGroupKeyGeneratorTest.java
@@ -46,11 +46,14 @@ import
org.apache.pinot.core.query.aggregation.groupby.NoDictionaryMultiColumnGr
import
org.apache.pinot.core.query.aggregation.groupby.NoDictionarySingleColumnGroupKeyGenerator;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.pql.parsers.Pql2Compiler;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.RecordReader;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -217,7 +220,9 @@ public class NoDictionaryGroupKeyGeneratorTest {
schema.addField(dimensionFieldSpec);
}
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(null, schema);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
+
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
config.setRawIndexCreationColumns(Arrays.asList(NO_DICT_COLUMN_NAMES));
config.setOutDir(INDEX_DIR_PATH);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/segments/v1/creator/OnHeapDictionariesTest.java
b/pinot-core/src/test/java/org/apache/pinot/segments/v1/creator/OnHeapDictionariesTest.java
index 303a455..a122dc4 100644
---
a/pinot-core/src/test/java/org/apache/pinot/segments/v1/creator/OnHeapDictionariesTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/segments/v1/creator/OnHeapDictionariesTest.java
@@ -28,6 +28,8 @@ import java.util.List;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.RandomStringUtils;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
@@ -43,6 +45,7 @@ import
org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.core.segment.index.readers.Dictionary;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -75,7 +78,9 @@ public class OnHeapDictionariesTest {
public void setup()
throws Exception {
Schema schema = buildSchema();
- buildSegment(SEGMENT_DIR_NAME, SEGMENT_NAME, schema);
+
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
+ buildSegment(SEGMENT_DIR_NAME, SEGMENT_NAME, tableConfig, schema);
IndexLoadingConfig loadingConfig = new IndexLoadingConfig();
loadingConfig.setReadMode(ReadMode.mmap);
@@ -163,10 +168,10 @@ public class OnHeapDictionariesTest {
* @return Schema built for the segment
* @throws Exception
*/
- private Schema buildSegment(String segmentDirName, String segmentName,
Schema schema)
+ private Schema buildSegment(String segmentDirName, String segmentName,
TableConfig tableConfig, Schema schema)
throws Exception {
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(null, schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
config.setOutDir(segmentDirName);
config.setFormat(FileFormat.AVRO);
config.setSegmentName(segmentName);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/segments/v1/creator/SegmentTestUtils.java
b/pinot-core/src/test/java/org/apache/pinot/segments/v1/creator/SegmentTestUtils.java
index c6d541d..ed801f6 100644
---
a/pinot-core/src/test/java/org/apache/pinot/segments/v1/creator/SegmentTestUtils.java
+++
b/pinot-core/src/test/java/org/apache/pinot/segments/v1/creator/SegmentTestUtils.java
@@ -37,12 +37,15 @@ import org.apache.commons.collections.Predicate;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.indexsegment.generator.SegmentVersion;
import org.apache.pinot.plugin.inputformat.avro.AvroSchemaUtil;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.MetricFieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.FileFormat;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,7 +58,8 @@ public class SegmentTestUtils {
@Nonnull File outputDir, @Nonnull String tableName)
throws IOException {
SegmentGeneratorConfig segmentGeneratorConfig =
- new SegmentGeneratorConfig(null,
extractSchemaFromAvroWithoutTime(avroFile));
+ new SegmentGeneratorConfig(new
TableConfigBuilder(TableType.OFFLINE).setTableName(tableName).build(),
+ extractSchemaFromAvroWithoutTime(avroFile));
segmentGeneratorConfig.setInputFilePath(avroFile.getAbsolutePath());
segmentGeneratorConfig.setOutDir(outputDir.getAbsolutePath());
segmentGeneratorConfig.setTableName(tableName);
@@ -66,7 +70,8 @@ public class SegmentTestUtils {
String timeColumn, TimeUnit timeUnit, String tableName)
throws IOException {
final SegmentGeneratorConfig segmentGenSpec =
- new SegmentGeneratorConfig(null,
extractSchemaFromAvroWithoutTime(inputAvro));
+ new SegmentGeneratorConfig(new
TableConfigBuilder(TableType.OFFLINE).setTableName(tableName).build(),
+ extractSchemaFromAvroWithoutTime(inputAvro));
segmentGenSpec.setInputFilePath(inputAvro.getAbsolutePath());
segmentGenSpec.setTimeColumnName(timeColumn);
segmentGenSpec.setSegmentTimeUnit(timeUnit);
@@ -79,8 +84,8 @@ public class SegmentTestUtils {
}
public static SegmentGeneratorConfig
getSegmentGeneratorConfigWithSchema(File inputAvro, File outputDir,
- String tableName, Schema schema) {
- SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(null, schema);
+ String tableName, TableConfig tableConfig, Schema schema) {
+ SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(tableConfig, schema);
segmentGeneratorConfig.setInputFilePath(inputAvro.getAbsolutePath());
segmentGeneratorConfig.setOutDir(outputDir.getAbsolutePath());
segmentGeneratorConfig.setFormat(FileFormat.AVRO);
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/DefaultCommitterRealtimeIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/DefaultCommitterRealtimeIntegrationTest.java
index adcb014..c2d4101 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/DefaultCommitterRealtimeIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/DefaultCommitterRealtimeIntegrationTest.java
@@ -41,6 +41,7 @@ import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.RecordReader;
import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.BeforeClass;
@@ -139,7 +140,8 @@ public class DefaultCommitterRealtimeIntegrationTest
extends RealtimeClusterInte
public void buildSegment()
throws Exception {
Schema schema = _helixResourceManager.getSchema(getTableName());
- TableConfig tableConfig =
_helixResourceManager.getTableConfig(getTableName());
+ TableConfig tableConfig = _helixResourceManager
+
.getTableConfig(TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(getTableName()));
String _segmentOutputDir = Files.createTempDir().toString();
List<GenericRow> _rows = PinotSegmentUtil.createTestData(schema, 1);
RecordReader _recordReader = new GenericRowRecordReader(_rows);
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/server/util/SegmentTestUtils.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/server/util/SegmentTestUtils.java
index 89d7daa..2a3e88f 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/server/util/SegmentTestUtils.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/server/util/SegmentTestUtils.java
@@ -24,10 +24,13 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.pinot.plugin.inputformat.avro.AvroUtils;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.FileFormat;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.indexsegment.generator.SegmentVersion;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
public class SegmentTestUtils {
@@ -38,12 +41,16 @@ public class SegmentTestUtils {
public static SegmentGeneratorConfig getSegmentGeneratorConfig(@Nonnull File
inputAvro, @Nonnull File outputDir,
@Nonnull TimeUnit timeUnit, @Nonnull String tableName, @Nullable Schema
pinotSchema)
throws IOException {
- SegmentGeneratorConfig segmentGeneratorConfig;
if (pinotSchema == null) {
- segmentGeneratorConfig = new SegmentGeneratorConfig(null,
AvroUtils.getPinotSchemaFromAvroDataFile(inputAvro));
- } else {
- segmentGeneratorConfig = new SegmentGeneratorConfig(null, pinotSchema);
+ pinotSchema = AvroUtils.getPinotSchemaFromAvroDataFile(inputAvro);
+ }
+ TableConfigBuilder tableConfigBuilder = new
TableConfigBuilder(TableType.OFFLINE).setTableName(tableName);
+ // TODO: change to getDateTimeFieldSpecs().get(0)
+ if (pinotSchema.getTimeFieldSpec() != null) {
+
tableConfigBuilder.setTimeColumnName(pinotSchema.getTimeFieldSpec().getName());
}
+ TableConfig tableConfig = tableConfigBuilder.build();
+ SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(tableConfig, pinotSchema);
segmentGeneratorConfig.setInputFilePath(inputAvro.getAbsolutePath());
segmentGeneratorConfig.setSegmentTimeUnit(timeUnit);
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/RawIndexBenchmark.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/RawIndexBenchmark.java
index 6d7abd9..0fddedc 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/RawIndexBenchmark.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/RawIndexBenchmark.java
@@ -28,6 +28,8 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
@@ -47,6 +49,7 @@ import
org.apache.pinot.core.operator.filter.TestFilterOperator;
import org.apache.pinot.core.plan.DocIdSetPlanNode;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.core.segment.creator.impl.V1Constants;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
@@ -127,8 +130,8 @@ public class RawIndexBenchmark {
DimensionFieldSpec dimensionFieldSpec = new DimensionFieldSpec(column,
FieldSpec.DataType.STRING, true);
schema.addField(dimensionFieldSpec);
}
-
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(null, schema);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
config.setRawIndexCreationColumns(Collections.singletonList(_rawIndexColumn));
config.setOutDir(SEGMENT_DIR_NAME);
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/StringDictionaryPerfTest.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/StringDictionaryPerfTest.java
index 6f585ac..9484cf9 100644
---
a/pinot-perf/src/main/java/org/apache/pinot/perf/StringDictionaryPerfTest.java
+++
b/pinot-perf/src/main/java/org/apache/pinot/perf/StringDictionaryPerfTest.java
@@ -29,6 +29,8 @@ import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
@@ -42,6 +44,7 @@ import
org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
import
org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.core.segment.index.readers.Dictionary;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
/**
@@ -77,11 +80,12 @@ public class StringDictionaryPerfTest {
FieldSpec fieldSpec = new DimensionFieldSpec(COLUMN_NAME,
FieldSpec.DataType.STRING, true);
schema.addField(fieldSpec);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
_dictLength = dictLength;
_inputStrings = new String[dictLength];
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(null, schema);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
config.setOutDir(_indexDir.getParent());
config.setFormat(FileFormat.AVRO);
config.setSegmentName(segmentName);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
