This is an automated email from the ASF dual-hosted git repository. sunithabeeram pushed a commit to branch createSegments in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit bdb264861c502b992c764c80ef6701efec0225b4 Author: Sunitha Beeram <[email protected]> AuthorDate: Tue May 28 10:38:18 2019 -0700 Add support for custom record-readers in the create-segment tool --- pinot-integration-tests/pom.xml | 6 ++++++ pinot-tools/pom.xml | 4 ++++ .../tools/admin/command/CreateSegmentCommand.java | 25 +++++++++++++--------- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/pinot-integration-tests/pom.xml b/pinot-integration-tests/pom.xml index 20ad7c8..01c0532 100644 --- a/pinot-integration-tests/pom.xml +++ b/pinot-integration-tests/pom.xml @@ -126,6 +126,12 @@ <dependency> <groupId>org.apache.pinot</groupId> <artifactId>pinot-tools</artifactId> + <exclusions> + <exclusion> + <groupId>org.apache.pinot</groupId> + <artifactId>pinot-orc</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.101tec</groupId> diff --git a/pinot-tools/pom.xml b/pinot-tools/pom.xml index 766852c..9fcd62e 100644 --- a/pinot-tools/pom.xml +++ b/pinot-tools/pom.xml @@ -44,6 +44,10 @@ </dependency> <dependency> <groupId>org.apache.pinot</groupId> + <artifactId>pinot-orc</artifactId> + </dependency> + <dependency> + <groupId>org.apache.pinot</groupId> <artifactId>pinot-server</artifactId> </dependency> <dependency> diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java index 95c1371..af22b87 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java @@ -228,15 +228,17 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co } } - FileFormat configFormat = segmentGeneratorConfig.getFormat(); - if (_format == null) { - if (configFormat == null) { - throw new RuntimeException("Format cannot be null in config file."); - } - _format = configFormat; - } else { - if (configFormat != _format && configFormat != FileFormat.AVRO) { - LOGGER.warn("Find format conflict in command line and config file, use config in command line: {}", _format); + if (segmentGeneratorConfig.getRecordReaderPath() == null) { + FileFormat configFormat = segmentGeneratorConfig.getFormat(); + if (_format == null) { + if (configFormat == null) { + throw new RuntimeException("Either recordReaderPath (via generatorConfigFile option) or format must be specified."); + } + _format = configFormat; + } else { + if (configFormat != _format && configFormat != FileFormat.AVRO) { + LOGGER.warn("Find format conflict in command line and config file, use config in command line: {}", _format); + } } } @@ -291,7 +293,10 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co File[] files = dir.listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { - return name.toLowerCase().endsWith(_format.toString().toLowerCase()); + if (_format != null) { + return name.toLowerCase().endsWith(_format.toString().toLowerCase()); + } + return true; } }); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
