This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch bugfixing_hadoop_filesystem
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/bugfixing_hadoop_filesystem by
this push:
new 89f6171 Adding more logs for file pattern match
89f6171 is described below
commit 89f6171f5cd8954a31cfa358b1facb489de9a838
Author: Xiang Fu <[email protected]>
AuthorDate: Mon Sep 30 01:59:33 2019 -0700
Adding more logs for file pattern match
---
.../main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java | 10 +++++++---
.../java/org/apache/pinot/hadoop/job/SegmentCreationJob.java | 2 +-
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java
index 7828fdf..005c90b 100644
--- a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java
+++ b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java
@@ -58,11 +58,15 @@ public abstract class BaseSegmentJob extends Configured {
return value != null ? new Path(value) : null;
}
- protected List<Path> getDataFilePaths(Path pathPattern)
+ protected List<Path> getDataFilePaths(FileSystem fileSystem, Path
pathPattern)
throws IOException {
List<Path> tarFilePaths = new ArrayList<>();
- FileSystem fileSystem = FileSystem.get(pathPattern.toUri(), _conf);
- getDataFilePathsHelper(fileSystem, fileSystem.globStatus(pathPattern),
tarFilePaths);
+ final FileStatus[] fileStatuses = fileSystem.globStatus(pathPattern);
+ if (fileStatuses == null) {
+ _logger.warn("Unable to match file status from file path pattern: {}",
pathPattern);
+ } else {
+ getDataFilePathsHelper(fileSystem, fileStatuses, tarFilePaths);
+ }
return tarFilePaths;
}
diff --git
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
index aa68f10..8a5b391 100644
---
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
+++
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
@@ -124,7 +124,7 @@ public class SegmentCreationJob extends BaseSegmentJob {
JobPreparationHelper.mkdirs(_fileSystem, stagingInputDir,
_defaultPermissionsMask);
// Gather all data files
- List<Path> dataFilePaths = getDataFilePaths(_inputPattern);
+ List<Path> dataFilePaths = getDataFilePaths(_fileSystem, _inputPattern);
int numDataFiles = dataFilePaths.size();
if (numDataFiles == 0) {
String errorMessage = "No data file founded with pattern: " +
_inputPattern;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
