This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a commit to branch new-dist-assembly in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 601c2ff7c57b88ccdada5a4e4fc0b76618a082da Author: Xiang Fu <[email protected]> AuthorDate: Wed Jan 1 17:39:10 2020 -0800 Move shaded jars to distribution plugins directory --- pinot-distribution/pinot-assembly.xml | 5 ++++ pinot-distribution/pom.xml | 10 ++++++++ pinot-plugins/pinot-batch-ingestion/pom.xml | 27 +++++++++++++++++++++ pinot-plugins/pinot-file-system/pom.xml | 27 +++++++++++++++++++++ pinot-plugins/pinot-input-format/pom.xml | 28 +++++++++++++++++++++- pinot-plugins/pinot-stream-ingestion/pom.xml | 27 +++++++++++++++++++++ pinot-plugins/pom.xml | 27 --------------------- pinot-tools/pom.xml | 25 +++++++++++-------- .../tools/admin/command/CreateSegmentCommand.java | 12 ++++------ 9 files changed, 142 insertions(+), 46 deletions(-) diff --git a/pinot-distribution/pinot-assembly.xml b/pinot-distribution/pinot-assembly.xml index 0c21f70..1aad15e 100644 --- a/pinot-distribution/pinot-assembly.xml +++ b/pinot-distribution/pinot-assembly.xml @@ -102,5 +102,10 @@ <outputDirectory>bin</outputDirectory> <fileMode>0755</fileMode> </fileSet> + <fileSet> + <useDefaultExcludes>false</useDefaultExcludes> + <directory>${pinot.root}/pinot-plugins/target/plugins/</directory> + <outputDirectory>plugins</outputDirectory> + </fileSet> </fileSets> </assembly> diff --git a/pinot-distribution/pom.xml b/pinot-distribution/pom.xml index 372ce92..0e96471 100644 --- a/pinot-distribution/pom.xml +++ b/pinot-distribution/pom.xml @@ -63,6 +63,16 @@ <dependency> <groupId>org.apache.pinot</groupId> <artifactId>pinot-tools</artifactId> + <exclusions> + <exclusion> + <groupId>org.apache.pinot</groupId> + <artifactId>pinot-csv</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.pinot</groupId> + <artifactId>pinot-kafka-${kafka.version}</artifactId> + </exclusion> + </exclusions> </dependency> </dependencies> <build> diff --git a/pinot-plugins/pinot-batch-ingestion/pom.xml b/pinot-plugins/pinot-batch-ingestion/pom.xml index 4a94aca..9b2b775 100644 --- a/pinot-plugins/pinot-batch-ingestion/pom.xml +++ b/pinot-plugins/pinot-batch-ingestion/pom.xml @@ -55,5 +55,32 @@ <scope>test</scope> </dependency> </dependencies> + <profiles> + <profile> + <id>build-shaded-jar</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <artifactId>maven-shade-plugin</artifactId> + <version>3.2.1</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <outputDirectory>${pinot.root}/pinot-plugins/target/plugins/pinot-batch-ingestion/${artifactId}</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> diff --git a/pinot-plugins/pinot-file-system/pom.xml b/pinot-plugins/pinot-file-system/pom.xml index 6833af5..3c96efa 100644 --- a/pinot-plugins/pinot-file-system/pom.xml +++ b/pinot-plugins/pinot-file-system/pom.xml @@ -81,5 +81,32 @@ <scope>test</scope> </dependency> </dependencies> + <profiles> + <profile> + <id>build-shaded-jar</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <artifactId>maven-shade-plugin</artifactId> + <version>3.2.1</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <outputDirectory>${pinot.root}/pinot-plugins/target/plugins/pinot-file-system/${artifactId}</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> diff --git a/pinot-plugins/pinot-input-format/pom.xml b/pinot-plugins/pinot-input-format/pom.xml index d4a2af4..4fc2d57 100644 --- a/pinot-plugins/pinot-input-format/pom.xml +++ b/pinot-plugins/pinot-input-format/pom.xml @@ -54,5 +54,31 @@ <scope>test</scope> </dependency> </dependencies> - + <profiles> + <profile> + <id>build-shaded-jar</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <artifactId>maven-shade-plugin</artifactId> + <version>3.2.1</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <outputDirectory>${pinot.root}/pinot-plugins/target/plugins/pinot-input-format/${artifactId}</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> diff --git a/pinot-plugins/pinot-stream-ingestion/pom.xml b/pinot-plugins/pinot-stream-ingestion/pom.xml index 6069a6d..662377e 100644 --- a/pinot-plugins/pinot-stream-ingestion/pom.xml +++ b/pinot-plugins/pinot-stream-ingestion/pom.xml @@ -80,5 +80,32 @@ <scope>test</scope> </dependency> </dependencies> + <profiles> + <profile> + <id>build-shaded-jar</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <artifactId>maven-shade-plugin</artifactId> + <version>3.2.1</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <outputDirectory>${pinot.root}/pinot-plugins/target/plugins/pinot-stream-ingestion/${artifactId}</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> diff --git a/pinot-plugins/pom.xml b/pinot-plugins/pom.xml index 716e5e8..749cc5f 100644 --- a/pinot-plugins/pom.xml +++ b/pinot-plugins/pom.xml @@ -56,31 +56,4 @@ <scope>test</scope> </dependency> </dependencies> - <profiles> - <profile> - <id>build-shaded-jar</id> - <activation> - <activeByDefault>true</activeByDefault> - </activation> - <build> - <plugins> - <plugin> - <artifactId>maven-shade-plugin</artifactId> - <version>3.2.1</version> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <outputDirectory>target/plugins/${artifactId}</outputDirectory> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> </project> diff --git a/pinot-tools/pom.xml b/pinot-tools/pom.xml index c9a8267..cb86fc4 100644 --- a/pinot-tools/pom.xml +++ b/pinot-tools/pom.xml @@ -56,16 +56,6 @@ </dependency> <dependency> <groupId>org.apache.pinot</groupId> - <artifactId>pinot-orc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.pinot</groupId> - <artifactId>pinot-parquet</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.pinot</groupId> <artifactId>pinot-csv</artifactId> <version>${project.version}</version> </dependency> @@ -81,6 +71,21 @@ <scope>runtime</scope> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-math3</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> </dependency> diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java index b948f05..27f5138 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java @@ -46,8 +46,6 @@ import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig; import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment; import org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader; import org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl; -import org.apache.pinot.orc.data.readers.ORCRecordReader; -import org.apache.pinot.parquet.data.readers.ParquetRecordReader; import org.apache.pinot.startree.hll.HllConfig; import org.apache.pinot.startree.hll.HllConstants; import org.apache.pinot.tools.Command; @@ -412,14 +410,12 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co final SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl(); switch (config.getFormat()) { case PARQUET: - RecordReader parquetRecordReader = new ParquetRecordReader(); - parquetRecordReader.init(new File(localFile), schema, null); - driver.init(config, parquetRecordReader); + config.setRecordReaderPath("org.apache.pinot.parquet.data.readers.ParquetRecordReader"); + driver.init(config); break; case ORC: - RecordReader orcRecordReader = new ORCRecordReader(); - orcRecordReader.init(new File(localFile), schema, null); - driver.init(config, orcRecordReader); + config.setRecordReaderPath("org.apache.pinot.orc.data.readers.ORCRecordReader"); + driver.init(config); break; case CSV: RecordReader csvRecordReader = new CSVRecordReader(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
