This is an automated email from the ASF dual-hosted git repository.
sseth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new 9f5c1b7 TEZ-4086. Allow various examples to work when outputPath is
on a FileSystem other than the default FileSystem. (#45)
9f5c1b7 is described below
commit 9f5c1b7f6d09884131aa2bab4e01e98768347def
Author: Siddharth Seth <[email protected]>
AuthorDate: Sun Oct 6 10:15:07 2019 -0700
TEZ-4086. Allow various examples to work when outputPath is on a FileSystem
other than the default FileSystem. (#45)
---
.../src/main/java/org/apache/tez/examples/HashJoinExample.java | 3 ++-
.../src/main/java/org/apache/tez/examples/JoinDataGen.java | 10 ++++++----
.../java/org/apache/tez/examples/SortMergeJoinExample.java | 3 ++-
.../org/apache/tez/mapreduce/examples/FilterLinesByWord.java | 6 +++++-
.../apache/tez/mapreduce/examples/TestOrderedWordCount.java | 8 ++++----
.../java/org/apache/tez/mapreduce/examples/UnionExample.java | 5 ++++-
tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java | 7 +++----
7 files changed, 26 insertions(+), 16 deletions(-)
diff --git
a/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java
b/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java
index 935ccbc..e762ac1 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java
@@ -113,7 +113,8 @@ public class HashJoinExample extends TezExampleBase {
Path outputPath = new Path(outputDir);
// Verify output path existence
- FileSystem fs = FileSystem.get(tezConf);
+ FileSystem fs = outputPath.getFileSystem(tezConf);
+ outputPath = fs.makeQualified(outputPath);
if (fs.exists(outputPath)) {
System.err.println("Output directory: " + outputDir + " already exists");
return 3;
diff --git
a/tez-examples/src/main/java/org/apache/tez/examples/JoinDataGen.java
b/tez-examples/src/main/java/org/apache/tez/examples/JoinDataGen.java
index 4c0d201..d5b784b 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/JoinDataGen.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/JoinDataGen.java
@@ -102,10 +102,10 @@ public class JoinDataGen extends TezExampleBase {
Path expectedOutputPath = new Path(expectedOutputDir);
// Verify output path existence
- FileSystem fs = FileSystem.get(tezConf);
int res = 0;
- res = checkOutputDirectory(fs, largeOutPath) + checkOutputDirectory(fs,
smallOutPath)
- + checkOutputDirectory(fs, expectedOutputPath);
+ res = checkOutputDirectory(tezConf, largeOutPath)
+ + checkOutputDirectory(tezConf, smallOutPath)
+ + checkOutputDirectory(tezConf, expectedOutputPath);
if (res != 0) {
return 3;
}
@@ -279,7 +279,9 @@ public class JoinDataGen extends TezExampleBase {
}
- private int checkOutputDirectory(FileSystem fs, Path path) throws
IOException {
+ private int checkOutputDirectory(Configuration conf, Path path) throws
IOException {
+ FileSystem fs = path.getFileSystem(conf);
+ path = fs.makeQualified(path);
if (fs.exists(path)) {
System.err.println("Output directory: " + path + " already exists");
return 2;
diff --git
a/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
b/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
index 1054e00..820aaa5 100644
---
a/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
+++
b/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
@@ -105,7 +105,8 @@ public class SortMergeJoinExample extends TezExampleBase {
Path outputPath = new Path(outputDir);
// Verify output path existence
- FileSystem fs = FileSystem.get(tezConf);
+ FileSystem fs = outputPath.getFileSystem(tezConf);
+ outputPath = fs.makeQualified(outputPath);
if (fs.exists(outputPath)) {
System.err.println("Output directory: " + outputDir + " already exists");
return 3;
diff --git
a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
index 36215a4..da4d9ea 100644
---
a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++
b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -124,7 +124,11 @@ public class FilterLinesByWord extends Configured
implements Tool {
String filterWord = otherArgs[2];
FileSystem fs = FileSystem.get(conf);
- if (fs.exists(new Path(outputPath))) {
+
+ Path outputPathAsPath = new Path(outputPath);
+ FileSystem outputFs = outputPathAsPath.getFileSystem(conf);
+ outputPathAsPath = outputFs.makeQualified(outputPathAsPath);
+ if (outputFs.exists(outputPathAsPath)) {
System.err.println("Output directory : " + outputPath + " already
exists");
return 2;
}
diff --git
a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
index 51e4be1..1b87e11 100644
---
a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
+++
b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
@@ -18,7 +18,6 @@
package org.apache.tez.mapreduce.examples;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
@@ -441,8 +440,6 @@ public class TestOrderedWordCount extends Configured
implements Tool {
HadoopShim hadoopShim = new HadoopShimsLoader(tezConf).getHadoopShim();
TestOrderedWordCount instance = new TestOrderedWordCount();
- FileSystem fs = FileSystem.get(conf);
-
String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR,
TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR +
Long.toString(System.currentTimeMillis());
@@ -498,7 +495,10 @@ public class TestOrderedWordCount extends Configured
implements Tool {
String inputPath = inputPaths.get(dagIndex-1);
String outputPath = outputPaths.get(dagIndex-1);
- if (fs.exists(new Path(outputPath))) {
+ Path outputPathAsPath = new Path(outputPath);
+ FileSystem fs = outputPathAsPath.getFileSystem(conf);
+ outputPathAsPath = fs.makeQualified(outputPathAsPath);
+ if (fs.exists(outputPathAsPath)) {
throw new FileAlreadyExistsException("Output directory "
+ outputPath + " already exists");
}
diff --git
a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
index 7688335..f78d162 100644
---
a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
+++
b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
@@ -262,7 +262,10 @@ public class UnionExample {
DAGClient dagClient = null;
try {
- if (fs.exists(new Path(outputPath))) {
+ Path outputPathAsPath = new Path(outputPath);
+ FileSystem outputFs = outputPathAsPath.getFileSystem(tezConf);
+ outputPathAsPath = outputFs.makeQualified(outputPathAsPath);
+ if (outputFs.exists(outputPathAsPath)) {
throw new FileAlreadyExistsException("Output directory "
+ outputPath + " already exists");
}
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
index 2dfc76d..2a94d9b 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
@@ -138,7 +138,6 @@ public class TestTezJobs {
if (mrrTezCluster == null) {
mrrTezCluster = new MiniTezCluster(TestTezJobs.class.getName(), 1, 1, 1);
- Configuration conf = new Configuration();
conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
conf.setLong(TezConfiguration.TEZ_AM_SLEEP_TIME_BEFORE_EXIT_MILLIS, 500);
mrrTezCluster.init(conf);
@@ -163,7 +162,7 @@ public class TestTezJobs {
@Test(timeout = 60000)
public void testHashJoinExample() throws Exception {
HashJoinExample hashJoinExample = new HashJoinExample();
- hashJoinExample.setConf(mrrTezCluster.getConfig());
+ hashJoinExample.setConf(new Configuration(mrrTezCluster.getConfig()));
Path stagingDirPath = new Path("/tmp/tez-staging-dir");
Path inPath1 = new Path("/tmp/hashJoin/inPath1");
Path inPath2 = new Path("/tmp/hashJoin/inPath2");
@@ -219,7 +218,7 @@ public class TestTezJobs {
@Test(timeout = 60000)
public void testHashJoinExampleDisableSplitGrouping() throws Exception {
HashJoinExample hashJoinExample = new HashJoinExample();
- hashJoinExample.setConf(conf);
+ hashJoinExample.setConf(new Configuration(mrrTezCluster.getConfig()));
Path stagingDirPath = new Path(TEST_ROOT_DIR + "/tmp/tez-staging-dir");
Path inPath1 = new Path(TEST_ROOT_DIR + "/tmp/hashJoin/inPath1");
Path inPath2 = new Path(TEST_ROOT_DIR + "/tmp/hashJoin/inPath2");
@@ -431,7 +430,7 @@ public class TestTezJobs {
@Test(timeout = 60000)
public void testSortMergeJoinExampleDisableSplitGrouping() throws Exception {
SortMergeJoinExample sortMergeJoinExample = new SortMergeJoinExample();
- sortMergeJoinExample.setConf(conf);
+ sortMergeJoinExample.setConf(new Configuration(mrrTezCluster.getConfig()));
Path stagingDirPath = new Path(TEST_ROOT_DIR + "/tmp/tez-staging-dir");
Path inPath1 = new Path(TEST_ROOT_DIR + "/tmp/sortMerge/inPath1");
Path inPath2 = new Path(TEST_ROOT_DIR + "/tmp/sortMerge/inPath2");
