This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a commit to branch refactor-record-reader in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 5942774220df244dc25a93d5cc97dd78b2e618a2 Author: kishoreg <[email protected]> AuthorDate: Tue Nov 26 22:49:09 2019 -0800 Removing segment generation config from RecordReader interface --- .../apache/pinot/core/data/readers/AvroRecordReader.java | 3 ++- .../apache/pinot/core/data/readers/CSVRecordReader.java | 3 ++- .../pinot/core/data/readers/GenericRowRecordReader.java | 3 ++- .../apache/pinot/core/data/readers/JSONRecordReader.java | 4 +++- .../data/readers/MultiplePinotSegmentRecordReader.java | 11 ++++++----- .../pinot/core/data/readers/PinotSegmentRecordReader.java | 11 ++++++----- .../org/apache/pinot/core/data/readers/RecordReader.java | 14 ++++++++++---- .../pinot/core/data/readers/RecordReaderFactory.java | 2 +- .../apache/pinot/core/data/readers/ThriftRecordReader.java | 11 ++++++----- .../indexsegment/generator/SegmentGeneratorConfig.java | 2 +- .../apache/pinot/core/minion/BackfillDateTimeColumn.java | 4 +++- .../java/org/apache/pinot/core/minion/SegmentPurger.java | 5 ++++- .../pinot/core/minion/segment/MapperRecordReader.java | 4 +++- .../pinot/core/minion/segment/ReducerRecordReader.java | 4 +++- .../realtime/converter/RealtimeSegmentRecordReader.java | 4 +++- .../org/apache/pinot/orc/data/readers/ORCRecordReader.java | 5 +++-- .../apache/pinot/orc/data/readers/ORCRecordReaderTest.java | 11 ++--------- .../pinot/parquet/data/readers/ParquetRecordReader.java | 7 ++++--- .../parquet/data/readers/ParquetRecordReaderTest.java | 6 +----- .../pinot/tools/admin/command/CreateSegmentCommand.java | 5 +++-- 20 files changed, 68 insertions(+), 51 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/AvroRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/AvroRecordReader.java index bfa225f..9d0fa4e 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/AvroRecordReader.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/AvroRecordReader.java @@ -56,7 +56,8 @@ public class AvroRecordReader implements RecordReader { } @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) { + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) + throws Exception { } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/CSVRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/CSVRecordReader.java index 00cce5d..95ac10a 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/CSVRecordReader.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/CSVRecordReader.java @@ -94,7 +94,8 @@ public class CSVRecordReader implements RecordReader { } @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) { + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) + throws Exception { } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/GenericRowRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/GenericRowRecordReader.java index 62466e2..e1f4ca2 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/GenericRowRecordReader.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/GenericRowRecordReader.java @@ -41,7 +41,8 @@ public class GenericRowRecordReader implements RecordReader { } @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) { + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) + throws Exception { } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/JSONRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/JSONRecordReader.java index cc4bcd7..459a167 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/JSONRecordReader.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/JSONRecordReader.java @@ -62,7 +62,9 @@ public class JSONRecordReader implements RecordReader { } @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) { + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) + throws Exception { + } @Override diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/MultiplePinotSegmentRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/MultiplePinotSegmentRecordReader.java index d924152..dda5fd9 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/MultiplePinotSegmentRecordReader.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/MultiplePinotSegmentRecordReader.java @@ -53,11 +53,6 @@ public class MultiplePinotSegmentRecordReader implements RecordReader { this(indexDirs, null, null); } - @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) { - - } - /** * Read records using the passed in schema and in the order of sorted column from multiple pinot segments. * <p>Passed in schema must be a subset of the segment schema. @@ -106,6 +101,12 @@ public class MultiplePinotSegmentRecordReader implements RecordReader { } } + @Override + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) + throws Exception { + + } + /** * Indicate whether the segment should be sorted or not */ diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/PinotSegmentRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/PinotSegmentRecordReader.java index 50c1808..fab4af7 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/PinotSegmentRecordReader.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/PinotSegmentRecordReader.java @@ -60,11 +60,6 @@ public class PinotSegmentRecordReader implements RecordReader { this(indexDir, null, null); } - @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) { - - } - /** * Read records using the segment schema with the given schema and sort order * <p>Passed in schema must be a subset of the segment schema. @@ -110,6 +105,12 @@ public class PinotSegmentRecordReader implements RecordReader { } } + @Override + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) + throws Exception { + + } + /** * Prepare sorted docIds in order of the given sort order columns */ diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReader.java index 836d3d8..7ea744e 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReader.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReader.java @@ -20,9 +20,9 @@ package org.apache.pinot.core.data.readers; import java.io.Closeable; import java.io.IOException; +import javax.annotation.Nullable; import org.apache.pinot.common.data.Schema; import org.apache.pinot.core.data.GenericRow; -import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig; /** @@ -34,10 +34,16 @@ import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig; public interface RecordReader extends Closeable { /** - * Initializes the record reader when needed + * initializing recordreader with inputpath, schema and recordreader config. <br/> + * The implementation can chose to ignore one or more of these parameters and handle null gracefully <br/> + * + * @param inputPath absolute path to the file/directory + * @param schema Pinot Schema associated with the table + * @param recordReaderConfig config for the reader specific to the format. e.g. delimiter for csv format etc + * @throws Exception if the arguments are invalid */ - void init(SegmentGeneratorConfig segmentGeneratorConfig) - throws IOException; + void init(@Nullable String inputPath, @Nullable Schema schema, @Nullable RecordReaderConfig recordReaderConfig) + throws Exception; /** * Return <code>true</code> if more records remain to be read. diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReaderFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReaderFactory.java index 1d63b15..03edcfa 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReaderFactory.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReaderFactory.java @@ -52,7 +52,7 @@ public class RecordReaderFactory { .warn("Using class: {} to read segment, ignoring configured file format: {}", recordReaderPath, fileFormat); } RecordReader recordReader = (RecordReader) Class.forName(recordReaderPath).newInstance(); - recordReader.init(segmentGeneratorConfig); + recordReader.init(dataFile.getAbsolutePath(), schema, segmentGeneratorConfig.getReaderConfig()); return recordReader; } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/ThriftRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/ThriftRecordReader.java index 1f24010..7da8181 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/ThriftRecordReader.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/ThriftRecordReader.java @@ -66,6 +66,12 @@ public class ThriftRecordReader implements RecordReader { init(); } + @Override + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) + throws Exception { + + } + private void init() throws IOException { _inputStream = RecordReaderUtils.getBufferedInputStream(_dataFile); @@ -78,11 +84,6 @@ public class ThriftRecordReader implements RecordReader { } } - @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) { - - } - private boolean hasMoreToRead() throws IOException { _inputStream.mark(1); diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java index fba03b2..ec51f79 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java @@ -84,7 +84,7 @@ public class SegmentGeneratorConfig { private String _dataDir = null; private String _inputFilePath = null; private FileFormat _format = FileFormat.AVRO; - private String _recordReaderPath = null; + private String _recordReaderPath = null; //TODO: this should be renamed to recordReaderClass or even better removed private String _outDir = null; private boolean _overwrite = false; private String _tableName = null; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/BackfillDateTimeColumn.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/BackfillDateTimeColumn.java index 174826f..e485108 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/minion/BackfillDateTimeColumn.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/BackfillDateTimeColumn.java @@ -33,6 +33,7 @@ import org.apache.pinot.core.data.GenericRow; import org.apache.pinot.core.data.readers.FileFormat; import org.apache.pinot.core.data.readers.PinotSegmentRecordReader; import org.apache.pinot.core.data.readers.RecordReader; +import org.apache.pinot.core.data.readers.RecordReaderConfig; import org.apache.pinot.core.data.recordtransformer.CompositeTransformer; import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig; import org.apache.pinot.core.segment.creator.RecordReaderSegmentCreationDataSource; @@ -137,7 +138,8 @@ public class BackfillDateTimeColumn { } @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) { + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) + throws Exception { } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java index 35ae334..82ea567 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java @@ -32,6 +32,7 @@ import org.apache.pinot.common.segment.StarTreeMetadata; import org.apache.pinot.core.data.GenericRow; import org.apache.pinot.core.data.readers.PinotSegmentRecordReader; import org.apache.pinot.core.data.readers.RecordReader; +import org.apache.pinot.core.data.readers.RecordReaderConfig; import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig; import org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl; import org.apache.pinot.core.segment.index.SegmentMetadataImpl; @@ -170,7 +171,9 @@ public class SegmentPurger { } @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) { + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) + throws Exception { + } @Override diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/MapperRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/MapperRecordReader.java index b27dff8..07ffd51 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/MapperRecordReader.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/MapperRecordReader.java @@ -26,6 +26,7 @@ import org.apache.pinot.common.data.Schema; import org.apache.pinot.core.data.GenericRow; import org.apache.pinot.core.data.readers.MultiplePinotSegmentRecordReader; import org.apache.pinot.core.data.readers.RecordReader; +import org.apache.pinot.core.data.readers.RecordReaderConfig; import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig; @@ -54,7 +55,8 @@ public class MapperRecordReader implements RecordReader { } @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) { + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) + throws Exception { } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/ReducerRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/ReducerRecordReader.java index ddf7b81..9a2d133 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/ReducerRecordReader.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/ReducerRecordReader.java @@ -26,6 +26,7 @@ import org.apache.pinot.common.data.Schema; import org.apache.pinot.core.data.GenericRow; import org.apache.pinot.core.data.readers.PinotSegmentRecordReader; import org.apache.pinot.core.data.readers.RecordReader; +import org.apache.pinot.core.data.readers.RecordReaderConfig; import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig; @@ -51,7 +52,8 @@ public class ReducerRecordReader implements RecordReader { } @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) { + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) + throws Exception { } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/converter/RealtimeSegmentRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/realtime/converter/RealtimeSegmentRecordReader.java index 235ed00..d8e7c84 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/realtime/converter/RealtimeSegmentRecordReader.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/realtime/converter/RealtimeSegmentRecordReader.java @@ -21,6 +21,7 @@ package org.apache.pinot.core.realtime.converter; import org.apache.pinot.common.data.Schema; import org.apache.pinot.core.data.GenericRow; import org.apache.pinot.core.data.readers.RecordReader; +import org.apache.pinot.core.data.readers.RecordReaderConfig; import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig; import org.apache.pinot.core.indexsegment.mutable.MutableSegmentImpl; @@ -51,7 +52,8 @@ public class RealtimeSegmentRecordReader implements RecordReader { } @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) { + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) + throws Exception { } diff --git a/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java b/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java index 7bfcf60..e9f982c 100644 --- a/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java +++ b/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java @@ -45,6 +45,7 @@ import org.apache.orc.mapred.OrcMapredRecordReader; import org.apache.pinot.common.data.Schema; import org.apache.pinot.core.data.GenericRow; import org.apache.pinot.core.data.readers.RecordReader; +import org.apache.pinot.core.data.readers.RecordReaderConfig; import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,9 +84,9 @@ public class ORCRecordReader implements RecordReader { } @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) throws IOException { - init(segmentGeneratorConfig.getInputFilePath(), segmentGeneratorConfig.getSchema()); + init(inputPath, schema); } @Override diff --git a/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java b/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java index b3282ed..bd80ad1 100644 --- a/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java +++ b/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java @@ -114,15 +114,12 @@ public class ORCRecordReaderTest { throws IOException { ORCRecordReader orcRecordReader = new ORCRecordReader(); - SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(); - segmentGeneratorConfig.setInputFilePath(ORC_FILE.getAbsolutePath()); Schema schema = new Schema(); FieldSpec xFieldSpec = new DimensionFieldSpec("x", FieldSpec.DataType.LONG, true); schema.addField(xFieldSpec); FieldSpec yFieldSpec = new DimensionFieldSpec("y", FieldSpec.DataType.BYTES, true); schema.addField(yFieldSpec); - segmentGeneratorConfig.setSchema(schema); - orcRecordReader.init(segmentGeneratorConfig); + orcRecordReader.init(ORC_FILE.getAbsolutePath(), schema, null); List<GenericRow> genericRows = new ArrayList<>(); while (orcRecordReader.hasNext()) { @@ -140,16 +137,12 @@ public class ORCRecordReaderTest { @Test public void testReadMVData() throws IOException{ ORCRecordReader orcRecordReader = new ORCRecordReader(); - - SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(); - segmentGeneratorConfig.setInputFilePath(MULTIVALUE_ORC_FILE.getAbsolutePath()); Schema schema = new Schema(); FieldSpec emailsFieldSpec = new DimensionFieldSpec("emails", FieldSpec.DataType.STRING, false); schema.addField(emailsFieldSpec); FieldSpec xFieldSpec = new DimensionFieldSpec("x", FieldSpec.DataType.INT, true); schema.addField(xFieldSpec); - segmentGeneratorConfig.setSchema(schema); - orcRecordReader.init(segmentGeneratorConfig); + orcRecordReader.init(MULTIVALUE_ORC_FILE.getAbsolutePath(), schema, null); List<GenericRow> genericRows = new ArrayList<>(); while (orcRecordReader.hasNext()) { diff --git a/pinot-parquet/src/main/java/org/apache/pinot/parquet/data/readers/ParquetRecordReader.java b/pinot-parquet/src/main/java/org/apache/pinot/parquet/data/readers/ParquetRecordReader.java index 8e597e2..e80a3f4 100644 --- a/pinot-parquet/src/main/java/org/apache/pinot/parquet/data/readers/ParquetRecordReader.java +++ b/pinot-parquet/src/main/java/org/apache/pinot/parquet/data/readers/ParquetRecordReader.java @@ -27,6 +27,7 @@ import org.apache.pinot.common.data.FieldSpec; import org.apache.pinot.common.data.Schema; import org.apache.pinot.core.data.GenericRow; import org.apache.pinot.core.data.readers.RecordReader; +import org.apache.pinot.core.data.readers.RecordReaderConfig; import org.apache.pinot.core.data.readers.RecordReaderUtils; import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig; import org.apache.pinot.core.util.AvroUtils; @@ -43,10 +44,10 @@ public class ParquetRecordReader implements RecordReader { private GenericRecord _nextRecord; @Override - public void init(SegmentGeneratorConfig segmentGeneratorConfig) + public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig) throws IOException { - _dataFilePath = new Path(segmentGeneratorConfig.getInputFilePath()); - _schema = segmentGeneratorConfig.getSchema(); + _dataFilePath = new Path(inputPath); + _schema = schema; AvroUtils.validateSchema(_schema, ParquetUtils.getParquetSchema(_dataFilePath)); _fieldSpecs = RecordReaderUtils.extractFieldSpecs(_schema); diff --git a/pinot-parquet/src/test/java/org/apache/pinot/parquet/data/readers/ParquetRecordReaderTest.java b/pinot-parquet/src/test/java/org/apache/pinot/parquet/data/readers/ParquetRecordReaderTest.java index 74822eb..cab095c 100644 --- a/pinot-parquet/src/test/java/org/apache/pinot/parquet/data/readers/ParquetRecordReaderTest.java +++ b/pinot-parquet/src/test/java/org/apache/pinot/parquet/data/readers/ParquetRecordReaderTest.java @@ -81,12 +81,8 @@ public class ParquetRecordReaderTest extends RecordReaderTest { @Test public void testParquetRecordReader() throws Exception { - SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(); - segmentGeneratorConfig.setInputFilePath(DATA_FILE.getAbsolutePath()); - segmentGeneratorConfig.setSchema(SCHEMA); - try (ParquetRecordReader recordReader = new ParquetRecordReader()) { - recordReader.init(segmentGeneratorConfig); + recordReader.init(DATA_FILE.getAbsolutePath(), SCHEMA, null); checkValue(recordReader); recordReader.rewind(); checkValue(recordReader); diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java index 0b5bfed..a1ed11a 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java @@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.pinot.common.data.Schema; import org.apache.pinot.common.data.StarTreeIndexSpec; import org.apache.pinot.common.segment.ReadMode; import org.apache.pinot.common.utils.JsonUtils; @@ -409,12 +410,12 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co switch (config.getFormat()) { case PARQUET: RecordReader parquetRecordReader = new ParquetRecordReader(); - parquetRecordReader.init(config); + parquetRecordReader.init(localFile, Schema.fromFile(new File(_schemaFile)), null); driver.init(config, parquetRecordReader); break; case ORC: RecordReader orcRecordReader = new ORCRecordReader(); - orcRecordReader.init(config); + orcRecordReader.init(localFile, Schema.fromFile(new File(_schemaFile)), null); driver.init(config, orcRecordReader); break; default: --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
