This is an automated email from the ASF dual-hosted git repository.
jenniferdai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1d02839 Adding flexible file paths (#3943)
1d02839 is described below
commit 1d02839dc8bae296befe44fcbebae0a198b80637
Author: Jennifer Dai <[email protected]>
AuthorDate: Mon Mar 11 14:28:18 2019 -0700
Adding flexible file paths (#3943)
* Currently, we enforce segments to have certain file extensions. If record
reader path is there, we will allow any extensions.
---
.../src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
index 5b3ecad..6939fd7 100644
---
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
+++
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
@@ -104,6 +104,10 @@ public class SegmentCreationJob extends BaseSegmentJob {
@Override
protected boolean isDataFile(String fileName) {
+ // Other files may have different extensions, eg: orc can have no extension
+ if (_properties.getProperty(JobConfigConstants.RECORD_READER_PATH) !=
null) {
+ return true;
+ }
return fileName.endsWith(".avro") || fileName.endsWith(".csv") ||
fileName.endsWith(".json") || fileName
.endsWith(".thrift");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
