This is an automated email from the ASF dual-hosted git repository.
asalamon74 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push:
new 0eb006f OOZIE-3574 JavaAction create incorrect fileSystem instance in
addActionLibs method (zuston via asalamon74)
0eb006f is described below
commit 0eb006f655e6fe92b471dcc76cd85f8329f7c8f2
Author: Andras Salamon <[email protected]>
AuthorDate: Fri Jan 31 13:33:42 2020 +0100
OOZIE-3574 JavaAction create incorrect fileSystem instance in addActionLibs
method (zuston via asalamon74)
---
.../oozie/action/hadoop/JavaActionExecutor.java | 4 +-
.../action/hadoop/ActionExecutorTestCase.java | 65 +++++++++++++---------
.../hadoop/TestJavaActionExecutorLibAddition.java | 53 +++++++++++++-----
release-log.txt | 1 +
4 files changed, 81 insertions(+), 42 deletions(-)
diff --git
a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 0940703..f4f1902 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -718,7 +718,7 @@ public class JavaActionExecutor extends ActionExecutor {
if (files == null) return;
for (FileStatus file : files) {
if (!shareLibExcluder.shouldExclude(file.getPath().toUri())) {
- addToCache(conf, appPath, file.getPath().toUri().getPath(),
false);
+ addToCache(conf, appPath, file.getPath().toString(), false);
}
}
}
@@ -734,7 +734,7 @@ public class JavaActionExecutor extends ActionExecutor {
Path actionLibsPath = new Path(actionLibsStr);
String user = conf.get("user.name");
FileSystem fs =
Services.get().get(HadoopAccessorService.class).createFileSystem(user,
- appPath.toUri(), conf);
+ actionLibsPath.toUri(), conf);
if (fs.exists(actionLibsPath)) {
addFilesToCacheIfNotExcluded(appPath, conf,
fs.listStatus(actionLibsPath));
}
diff --git
a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java
b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java
index 6f118bd..33c5382 100644
---
a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java
+++
b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java
@@ -18,6 +18,31 @@
package org.apache.oozie.action.hadoop;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.Writer;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -42,8 +67,8 @@ import org.apache.oozie.service.HadoopAccessorException;
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.LiteWorkflowStoreService;
import org.apache.oozie.service.Services;
-import org.apache.oozie.service.UUIDService;
import org.apache.oozie.service.ShareLibService;
+import org.apache.oozie.service.UUIDService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.service.WorkflowStoreService;
import org.apache.oozie.test.XHCatTestCase;
@@ -58,32 +83,6 @@ import org.apache.oozie.workflow.lite.EndNodeDef;
import org.apache.oozie.workflow.lite.LiteWorkflowApp;
import org.apache.oozie.workflow.lite.StartNodeDef;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.io.Reader;
-import java.io.StringReader;
-import java.io.Writer;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Collection;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.FileInputStream;
-
-import java.text.SimpleDateFormat;
-
public abstract class ActionExecutorTestCase extends XHCatTestCase {
protected static final int JOB_TIMEOUT = 100_000;
@@ -517,12 +516,24 @@ public abstract class ActionExecutorTestCase extends
XHCatTestCase {
}
}
+ void createFilesWithFs2(Collection<Path> paths) throws Exception{
+ for(Path p : paths){
+ getFileSystem2().create(p);
+ }
+ }
+
void makeDirs(Path... dirs) throws Exception{
for(Path p : dirs){
getFileSystem().mkdirs(p);
}
}
+ void makeDirWithFs2(Path... dirs) throws IOException {
+ for(Path p : dirs){
+ getFileSystem2().mkdirs(p);
+ }
+ }
+
private void assertContainsJarsOrNot(boolean contains, String
cacheFilesStr, Collection<Path> jars) {
for (Path jar : jars) {
assertEquals("Unexpected distributed cache file content",
contains, cacheFilesStr.contains(jar.toString()));
diff --git
a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutorLibAddition.java
b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutorLibAddition.java
index 00e7777..ed2e20c 100644
---
a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutorLibAddition.java
+++
b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutorLibAddition.java
@@ -18,6 +18,18 @@
package org.apache.oozie.action.hadoop;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.Path;
@@ -33,19 +45,6 @@ import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
import org.jdom.Element;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-
import static
org.apache.oozie.action.hadoop.JavaActionExecutor.ACTION_SHARELIB_FOR;
import static
org.apache.oozie.action.hadoop.JavaActionExecutor.SHARELIB_EXCLUDE_SUFFIX;
@@ -98,6 +97,12 @@ public class TestJavaActionExecutorLibAddition extends
ActionExecutorTestCase {
};
@Override
+ protected void beforeSetUp() throws Exception {
+ super.beforeSetUp();
+ setSystemProperty("oozie.test.hadoop.minicluster2", "true");
+ }
+
+ @Override
protected void setSystemProps() throws Exception {
super.setSystemProps();
setHadoopSystemProps();
@@ -303,6 +308,12 @@ public class TestJavaActionExecutorLibAddition extends
ActionExecutorTestCase {
return actionLibPath;
}
+ private Path getActionLibPathWithFs2() throws Exception {
+ Path actionLibPath = new Path(getFs2TestCaseDir(), "actionlibs");
+ makeDirWithFs2(actionLibPath);
+ return actionLibPath;
+ }
+
private List<Path> createTestActionLibPaths(Path... paths) throws
Exception{
final Path actionLibPath = new Path(getFsTestCaseDir(), "actionlibs");
makeDirs(actionLibPath);
@@ -310,6 +321,11 @@ public class TestJavaActionExecutorLibAddition extends
ActionExecutorTestCase {
return Arrays.asList(paths);
}
+ private List<Path> createTestActionLibPathsWithFs2(Path... paths) throws
Exception{
+ createFilesWithFs2(Arrays.asList(paths));
+ return Arrays.asList(paths);
+ }
+
public void testAddingActionLibDir() throws Exception{
makeDirs(getActionLibPath());
List<Path> expectedJars = createTestActionLibPaths(
@@ -321,6 +337,17 @@ public class TestJavaActionExecutorLibAddition extends
ActionExecutorTestCase {
assertContainsJars(getDistributedCacheFilesStr(jobConf), expectedJars);
}
+ public void testAddingActionLibDirWhenActionJarsWithFs2() throws Exception
{
+ Path baseActionLibPath = getActionLibPathWithFs2();
+ List<Path> expectedJars = createTestActionLibPathsWithFs2(
+ new Path(baseActionLibPath, "jar1.jar"),
+ new Path(baseActionLibPath, "jar2.jar"));
+
+ Context context = createContextUsingSharelib(baseActionLibPath);
+ Configuration jobConf =
createActionExecutorAndSetLibFilesArchives(context);
+ assertContainsJars(getDistributedCacheFilesStr(jobConf), expectedJars);
+ }
+
public void testAddingActionLibFile() throws Exception{
List<Path> expectedJars = createTestActionLibPaths(new
Path(getFsTestCaseDir(), "jar3.jar"));
diff --git a/release-log.txt b/release-log.txt
index 21f70f7..31eeda3 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.3.0 release (trunk - unreleased)
+OOZIE-3574 JavaAction create incorrect fileSystem instance in addActionLibs
method (zuston via asalamon74)
OOZIE-3569 SSH Action should add checking success file (zuston via asalamon74)
OOZIE-3305 Prometheus /metrics http endpoint for monitoring integration (qsbao
via gezapeti)
OOZIE-3575 Add credential support for cloud file systems (matijhs via gezapeti)
