This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a commit to branch bugfixing_hadoop_filesystem in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 7ae4bf4dd72b974f61aaab1e324f52383d911985 Author: Xiang Fu <[email protected]> AuthorDate: Mon Sep 30 10:37:50 2019 -0700 more logging --- .../src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java | 8 +++++++- .../main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java index 7828fdf..fb39ac1 100644 --- a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java +++ b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java @@ -62,7 +62,13 @@ public abstract class BaseSegmentJob extends Configured { throws IOException { List<Path> tarFilePaths = new ArrayList<>(); FileSystem fileSystem = FileSystem.get(pathPattern.toUri(), _conf); - getDataFilePathsHelper(fileSystem, fileSystem.globStatus(pathPattern), tarFilePaths); + _logger.info("Using filesystem: {}", fileSystem); + final FileStatus[] fileStatuses = fileSystem.globStatus(pathPattern); + if (fileStatuses == null) { + _logger.warn("Unable to match file status from file path pattern: {}", pathPattern); + } else { + getDataFilePathsHelper(fileSystem, fileStatuses, tarFilePaths); + } return tarFilePaths; } diff --git a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java index aa68f10..7d9c651 100644 --- a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java +++ b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java @@ -109,7 +109,7 @@ public class SegmentCreationJob extends BaseSegmentJob { return true; } return fileName.endsWith(".avro") || fileName.endsWith(".csv") || fileName.endsWith(".json") || fileName - .endsWith(".thrift"); + .endsWith(".thrift") || fileName.endsWith(".parquet"); } public void run() --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
