This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a commit to branch pinot-avro-dep-clean-up in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 317eae6084725b27d0dacfc1f92383ae1ab58b48 Author: kishoreg <[email protected]> AuthorDate: Tue Dec 17 15:11:26 2019 -0800 changing pinot-avro dependency scope to test from compile --- pinot-batch-ingestion/pinot-spark/pom.xml | 4 +++ .../pinot-connector-kafka-base/pom.xml | 4 +++ pinot-core/pom.xml | 11 ++++---- .../generator/SegmentGeneratorConfig.java | 32 +--------------------- .../impl/SegmentIndexCreationDriverImpl.java | 3 +- pinot-server/pom.xml | 5 ++++ pinot-tools/pom.xml | 5 ++++ .../tools/admin/command/CreateSegmentCommand.java | 9 +++++- 8 files changed, 35 insertions(+), 38 deletions(-) diff --git a/pinot-batch-ingestion/pinot-spark/pom.xml b/pinot-batch-ingestion/pinot-spark/pom.xml index f587bf8..37e3703 100644 --- a/pinot-batch-ingestion/pinot-spark/pom.xml +++ b/pinot-batch-ingestion/pinot-spark/pom.xml @@ -124,6 +124,10 @@ <groupId>org.apache.pinot</groupId> <artifactId>pinot-core</artifactId> </exclusion> + <exclusion> + <groupId>com.thoughtworks.paranamer</groupId> + <artifactId>paranamer</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/pinot-connectors/pinot-connector-kafka-base/pom.xml b/pinot-connectors/pinot-connector-kafka-base/pom.xml index faeafda..2a02620 100644 --- a/pinot-connectors/pinot-connector-kafka-base/pom.xml +++ b/pinot-connectors/pinot-connector-kafka-base/pom.xml @@ -43,5 +43,9 @@ <artifactId>testng</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.pinot</groupId> + <artifactId>pinot-avro</artifactId> + </dependency> </dependencies> </project> diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml index 741b569..94b9da1 100644 --- a/pinot-core/pom.xml +++ b/pinot-core/pom.xml @@ -66,11 +66,6 @@ </dependency> <dependency> <groupId>org.apache.pinot</groupId> - <artifactId>pinot-avro</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.pinot</groupId> <artifactId>pinot-common</artifactId> </dependency> <dependency> @@ -185,6 +180,12 @@ </dependency> <dependency> <groupId>org.apache.pinot</groupId> + <artifactId>pinot-avro</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.pinot</groupId> <artifactId>pinot-csv</artifactId> <version>${project.version}</version> <scope>test</scope> diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java index 775c048..2e30814 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java @@ -36,7 +36,6 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.commons.lang.StringUtils; -import org.apache.pinot.avro.data.readers.AvroUtils; import org.apache.pinot.common.config.IndexingConfig; import org.apache.pinot.common.config.SegmentPartitionConfig; import org.apache.pinot.common.config.SegmentsValidationAndRetentionConfig; @@ -620,36 +619,7 @@ public class SegmentGeneratorConfig { return getQualifyingFields(FieldType.METRIC, true); } - /** - * @deprecated Load outside the class and use the setter for schema setting. - * @throws IOException - */ - @Deprecated - public void loadConfigFiles() - throws IOException { - Schema schema; - if (_schemaFile != null) { - schema = Schema.fromFile(new File(_schemaFile)); - setSchema(schema); - } else if (_format == FileFormat.AVRO) { - schema = AvroUtils.getPinotSchemaFromAvroDataFile(new File(_inputFilePath)); - setSchema(schema); - } else { - throw new RuntimeException("Input format " + _format + " requires schema."); - } - - if (_readerConfigFile != null) { - try { - setReaderConfig(RecordReaderFactory.getRecordReaderConfig(FileFormat.CSV, _readerConfigFile)); - } catch (IOException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - - @JsonIgnore + @JsonIgnore public String getDimensions() { return getQualifyingFields(FieldType.DIMENSION, true); } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/SegmentIndexCreationDriverImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/SegmentIndexCreationDriverImpl.java index 7e0092f..29d01c1 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/SegmentIndexCreationDriverImpl.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/SegmentIndexCreationDriverImpl.java @@ -66,6 +66,7 @@ import org.apache.pinot.spi.data.Schema; import org.apache.pinot.spi.data.readers.FileFormat; import org.apache.pinot.spi.data.readers.GenericRow; import org.apache.pinot.spi.data.readers.RecordReader; +import org.apache.pinot.spi.data.readers.RecordReaderFactory; import org.apache.pinot.startree.hll.HllConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -127,7 +128,7 @@ public class SegmentIndexCreationDriverImpl implements SegmentIndexCreationDrive LOGGER.warn("Using class: {} to read segment, ignoring configured file format: {}", recordReaderClassName, fileFormat); } - return org.apache.pinot.spi.data.readers.RecordReaderFactory + return RecordReaderFactory .getRecordReaderByClass(recordReaderClassName, dataFile, schema, segmentGeneratorConfig.getReaderConfig()); } diff --git a/pinot-server/pom.xml b/pinot-server/pom.xml index d29bb30..504dc9d 100644 --- a/pinot-server/pom.xml +++ b/pinot-server/pom.xml @@ -225,6 +225,11 @@ </dependency> <!-- test --> <dependency> + <groupId>org.apache.pinot</groupId> + <artifactId>pinot-avro</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <scope>test</scope> diff --git a/pinot-tools/pom.xml b/pinot-tools/pom.xml index 9028a4a..2ac3394 100644 --- a/pinot-tools/pom.xml +++ b/pinot-tools/pom.xml @@ -66,6 +66,11 @@ </dependency> <dependency> <groupId>org.apache.pinot</groupId> + <artifactId>pinot-csv</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.pinot</groupId> <artifactId>pinot-connector-kafka-base</artifactId> <version>${project.version}</version> </dependency> diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java index f451a4a..176e925 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java @@ -34,6 +34,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.pinot.csv.data.readers.CSVRecordReader; +import org.apache.pinot.csv.data.readers.CSVRecordReaderConfig; import org.apache.pinot.spi.data.Schema; import org.apache.pinot.common.data.StarTreeIndexSpec; import org.apache.pinot.common.segment.ReadMode; @@ -404,7 +406,6 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co dataDirPath.getFileSystem(new Configuration()).copyToLocalFile(dataFilePath, localFilePath); config.setInputFilePath(localFile); config.setSegmentName(_segmentName + "_" + segCnt); - config.loadConfigFiles(); final SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl(); switch (config.getFormat()) { @@ -418,6 +419,12 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co orcRecordReader.init(new File(localFile), Schema.fromFile(new File(_schemaFile)), null); driver.init(config, orcRecordReader); break; + case CSV: + RecordReader csvRecordReader = new CSVRecordReader(); + CSVRecordReaderConfig readerConfig = JsonUtils.fileToObject(new File(_readerConfigFile), CSVRecordReaderConfig.class); + csvRecordReader.init(new File(localFile), Schema.fromFile(new File(_schemaFile)), readerConfig); + driver.init(config, csvRecordReader); + break; default: driver.init(config); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
