This is an automated email from the ASF dual-hosted git repository.
dionusos pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push:
new f85cb2228 OOZIE-3678 Reduce the number of NameNode access when
starting the Yarn job (jmakai via dionusos)
f85cb2228 is described below
commit f85cb22289cb706aa4ce57162ebf9b361c35b4e2
Author: Denes Bodo <[email protected]>
AuthorDate: Thu Dec 1 15:10:09 2022 +0100
OOZIE-3678 Reduce the number of NameNode access when starting the Yarn job
(jmakai via dionusos)
---
.../java/org/apache/oozie/util/ClasspathUtils.java | 11 +++--
core/src/main/resources/oozie-default.xml | 8 +++
.../org/apache/oozie/util/TestClasspathUtils.java | 57 ++++++++++++++++++++++
release-log.txt | 1 +
4 files changed, 73 insertions(+), 4 deletions(-)
diff --git a/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java
b/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java
index fc404b473..15f13e7d7 100644
--- a/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.oozie.service.ConfigurationService;
import java.io.File;
import java.io.IOException;
@@ -40,6 +41,7 @@ import java.util.Map;
public class ClasspathUtils {
+ public static final String OOZIE_CLASSPATHUTILS_RESOLVE =
"oozie.classpathutils.resolve";
private static boolean usingMiniYarnCluster = false;
private static final List<String> CLASSPATH_ENTRIES = Arrays.asList(
ApplicationConstants.Environment.PWD.$(),
@@ -109,13 +111,14 @@ public class ClasspathUtils {
Map<String, String> environment,
String classpathEnvVar) throws
IOException {
if (paths != null) {
+ boolean resolve =
ConfigurationService.getBoolean(OOZIE_CLASSPATHUTILS_RESOLVE);
HashMap<Path, String> linkLookup = new HashMap<Path, String>();
if (withLinks != null) {
for (URI u: withLinks) {
Path p = new Path(u);
FileSystem remoteFS = p.getFileSystem(conf);
- p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(),
- remoteFS.getWorkingDirectory()));
+ p = p.makeQualified(remoteFS.getUri(),
remoteFS.getWorkingDirectory());
+ if (resolve) p = remoteFS.resolvePath(p);
String name = (null == u.getFragment())
? p.getName() : u.getFragment();
if (!name.toLowerCase(Locale.ENGLISH).endsWith(".jar")) {
@@ -126,8 +129,8 @@ public class ClasspathUtils {
for (Path p : paths) {
FileSystem remoteFS = p.getFileSystem(conf);
- p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(),
- remoteFS.getWorkingDirectory()));
+ p = p.makeQualified(remoteFS.getUri(),
remoteFS.getWorkingDirectory());
+ if (resolve) p = remoteFS.resolvePath(p);
String name = linkLookup.get(p);
if (name == null) {
name = p.getName();
diff --git a/core/src/main/resources/oozie-default.xml
b/core/src/main/resources/oozie-default.xml
index cf5dc18d5..a11c2e753 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -3497,4 +3497,12 @@ will be the requeue interval for the actions which are
waiting for a long time w
</description>
</property>
+ <property>
+ <name>oozie.classpathutils.resolve</name>
+ <value>false</value>
+ <description>
+ If false, then Oozie will not turn to NameNode to resolve file
path one by one before starts the Yarn application.
+ </description>
+ </property>
+
</configuration>
diff --git a/core/src/test/java/org/apache/oozie/util/TestClasspathUtils.java
b/core/src/test/java/org/apache/oozie/util/TestClasspathUtils.java
index 2e732cbec..70b3201bf 100644
--- a/core/src/test/java/org/apache/oozie/util/TestClasspathUtils.java
+++ b/core/src/test/java/org/apache/oozie/util/TestClasspathUtils.java
@@ -23,14 +23,21 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.test.XFsTestCase;
import org.apache.oozie.test.XTestCase;
+import org.mockito.Mockito;
+import java.io.IOException;
+import java.lang.reflect.Method;
import java.net.URI;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
+import static
org.apache.oozie.util.ClasspathUtils.OOZIE_CLASSPATHUTILS_RESOLVE;
+
public class TestClasspathUtils extends XFsTestCase {
@Override
@@ -104,4 +111,54 @@ public class TestClasspathUtils extends XFsTestCase {
assertEquals("$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*", paths[0]);
assertEquals("$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*",
paths[1]);
}
+
+ public void testResolvePathNotInvoked() throws Exception {
+ // turning OFF resolving file path
+ ConfigurationService.setBoolean(OOZIE_CLASSPATHUTILS_RESOLVE, false);
+
+ Configuration conf = new Configuration(false);
+ Map<String, String> env = new HashMap<>();
+
+ Path p1 = new Path(getFsTestCaseDir(), "foo.xml");
+ Path p2 = new Path(getFsTestCaseDir(), "fooSymlink");
+ getFileSystem().createNewFile(p1);
+ getFileSystem().createSymlink(p1, p2, false);
+ DistributedCache.addFileToClassPath(p1, conf);
+ DistributedCache.addFileToClassPath(p2, conf);
+
+ ClasspathUtils.setupClasspath(env, conf);
+
+ assertEquals(2, env.size());
+ assertTrue(env.containsKey("$PWD"));
+ String[] paths = env.get("$PWD").split(":");
+ assertEquals(2, paths.length);
+ Arrays.sort(paths);
+ assertEquals("$PWD/foo.xml", paths[0]);
+ assertEquals("$PWD/fooSymlink", paths[1]);
+ }
+
+ public void testResolvePathInvoked() throws Exception {
+ // turning ON resolving file path
+ ConfigurationService.setBoolean(OOZIE_CLASSPATHUTILS_RESOLVE, true);
+
+ Configuration conf = new Configuration(false);
+ Map<String, String> env = new HashMap<>();
+
+ Path p1 = new Path(getFsTestCaseDir(), "foo.xml");
+ Path p2 = new Path(getFsTestCaseDir(), "fooSymlink");
+ getFileSystem().createNewFile(p1);
+ getFileSystem().createSymlink(p1, p2, false);
+ DistributedCache.addFileToClassPath(p1, conf);
+ DistributedCache.addFileToClassPath(p2, conf);
+
+ ClasspathUtils.setupClasspath(env, conf);
+
+ assertEquals(2, env.size());
+ assertTrue(env.containsKey("$PWD"));
+ String[] paths = env.get("$PWD").split(":");
+ assertEquals(2, paths.length);
+ Arrays.sort(paths);
+ assertEquals("$PWD/foo.xml", paths[0]);
+ assertEquals("$PWD/foo.xml", paths[1]);
+ }
}
diff --git a/release-log.txt b/release-log.txt
index 4cfb75bd5..9f73a0652 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.3.0 release (trunk - unreleased)
+OOZIE-3678 Reduce the number of NameNode access when starting the Yarn job
(jmakai via dionusos)
OOZIE-3670 Actions can stuck while running in a Fork-Join workflow (jmakai via
dionusos)
OOZIE-3676 Remove all non FIPS compliant encoding algorithms (jmakai via
dionusos)
OOZIE-3674 Add a --insecure like parameter to Oozie client so it can ignore
certificate errors (jmakai via dionusos)
