This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch bugfixing_hadoop_filesystem
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/bugfixing_hadoop_filesystem by
this push:
new 65b4b7e fixing compile
65b4b7e is described below
commit 65b4b7e07c84257f7009ae031e5528b70daf3ac8
Author: Xiang Fu <[email protected]>
AuthorDate: Mon Sep 30 02:35:09 2019 -0700
fixing compile
---
.../java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java | 2 +-
.../src/main/java/org/apache/pinot/hadoop/job/SegmentTarPushJob.java | 2 +-
.../src/main/java/org/apache/pinot/hadoop/job/SegmentUriPushJob.java | 4 +++-
3 files changed, 5 insertions(+), 3 deletions(-)
diff --git
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
index 942fb6d..53312f3 100644
---
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
+++
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
@@ -139,7 +139,7 @@ public class SegmentPreprocessingJob extends BaseSegmentJob
{
}
_fileSystem = FileSystem.get(_conf);
- final List<Path> inputDataPaths = getDataFilePaths(_inputSegmentDir);
+ final List<Path> inputDataPaths = getDataFilePaths(_fileSystem,
_inputSegmentDir);
Preconditions.checkState(inputDataPaths.size() != 0, "No files in the
input directory.");
if (_fileSystem.exists(_preprocessedOutputDir)) {
diff --git
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentTarPushJob.java
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentTarPushJob.java
index 5385aaf..4b8f320 100644
---
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentTarPushJob.java
+++
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentTarPushJob.java
@@ -56,7 +56,7 @@ public class SegmentTarPushJob extends BaseSegmentJob {
public void run()
throws Exception {
FileSystem fileSystem = FileSystem.get(_conf);
- List<Path> segmentsToPush = getDataFilePaths(_segmentPattern);
+ List<Path> segmentsToPush = getDataFilePaths(fileSystem, _segmentPattern);
try (ControllerRestApi controllerRestApi = getControllerRestApi()) {
// TODO: Deal with invalid prefixes in the future
diff --git
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentUriPushJob.java
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentUriPushJob.java
index 708fdbf..fd67494 100644
---
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentUriPushJob.java
+++
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentUriPushJob.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.pinot.hadoop.utils.PushLocation;
@@ -53,7 +54,8 @@ public class SegmentUriPushJob extends BaseSegmentJob {
public void run()
throws Exception {
try (ControllerRestApi controllerRestApi = getControllerRestApi()) {
- List<Path> tarFilePaths = getDataFilePaths(_segmentPattern);
+ FileSystem fileSystem = FileSystem.get(_conf);
+ List<Path> tarFilePaths = getDataFilePaths(fileSystem, _segmentPattern);
List<String> segmentUris = new ArrayList<>(tarFilePaths.size());
for (Path tarFilePath : tarFilePaths) {
segmentUris.add(_segmentUriPrefix + tarFilePath.toUri().getRawPath() +
_segmentUriSuffix);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
