Repository: oozie Updated Branches: refs/heads/master 3f7a0c562 -> 16234592b
OOZIE-2347 Remove unnecessary new Configuration()/new jobConf() calls from oozie Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/16234592 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/16234592 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/16234592 Branch: refs/heads/master Commit: 16234592b76b89aa1e7f5359e00d28ad138fe1ab Parents: 3f7a0c5 Author: Purshotam Shah <[email protected]> Authored: Tue Sep 1 14:45:16 2015 -0700 Committer: Purshotam Shah <[email protected]> Committed: Tue Sep 1 14:45:16 2015 -0700 ---------------------------------------------------------------------- .../org/apache/oozie/action/hadoop/JavaActionExecutor.java | 8 +++++--- .../java/org/apache/oozie/service/HadoopAccessorService.java | 8 ++++++-- .../main/java/org/apache/oozie/service/ShareLibService.java | 5 ++--- core/src/main/java/org/apache/oozie/util/JobUtils.java | 7 ++----- .../apache/oozie/action/hadoop/TestJavaActionExecutor.java | 2 +- release-log.txt | 1 + 6 files changed, 17 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/16234592/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index bb2d3a6..5eacffe 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -642,9 +642,11 @@ public class JavaActionExecutor extends ActionExecutor { if (confSet.contains(fileName)) { Configuration jobXmlConf = shareLibService.getShareLibConf(actionShareLibName, pathWithFragment); - checkForDisallowedProps(jobXmlConf, actionLibPath.getName()); - XConfiguration.injectDefaults(jobXmlConf, conf); - LOG.trace("Adding properties of " + actionLibPath + " to job conf"); + if (jobXmlConf != null) { + checkForDisallowedProps(jobXmlConf, actionLibPath.getName()); + XConfiguration.injectDefaults(jobXmlConf, conf); + LOG.trace("Adding properties of " + actionLibPath + " to job conf"); + } } else { // Filtering out duplicate jars or files http://git-wip-us.apache.org/repos/asf/oozie/blob/16234592/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java index b72b90f..116f382 100644 --- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java +++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java @@ -28,7 +28,6 @@ import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIden import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.security.token.Token; import org.apache.oozie.ErrorCode; import org.apache.oozie.action.hadoop.JavaActionExecutor; @@ -84,6 +83,8 @@ public class HadoopAccessorService implements Service { protected static final String HADOOP_YARN_RM = "yarn.resourcemanager.address"; private static final Map<String, Text> mrTokenRenewers = new HashMap<String, Text>(); + private static JobConf cachedJobConf; + private static final String DEFAULT_ACTIONNAME = "default"; private Set<String> jobTrackerWhitelist = new HashSet<String>(); @@ -146,6 +147,9 @@ public class HadoopAccessorService implements Service { loadHadoopConfigs(conf); preLoadActionConfigs(conf); + cachedJobConf = new JobConf(); + //calling .size to invoke lazy loading of hadoop properties + cachedJobConf.size(); supportedSchemes = new HashSet<String>(); String[] schemesFromConf = ConfigurationService.getStrings(conf, SUPPORTED_FILESYSTEMS); @@ -294,7 +298,7 @@ public class HadoopAccessorService implements Service { * @return a JobConf with the corresponding site configuration for hostPort. */ public JobConf createJobConf(String hostPort) { - JobConf jobConf = new JobConf(); + JobConf jobConf = new JobConf(cachedJobConf); XConfiguration.copy(getConfiguration(hostPort), jobConf); jobConf.setBoolean(OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED, true); return jobConf; http://git-wip-us.apache.org/repos/asf/oozie/blob/16234592/core/src/main/java/org/apache/oozie/service/ShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java index b783379..81a9f2d 100644 --- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java +++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java @@ -862,12 +862,11 @@ public class ShareLibService implements Service, Instrumentable { } public Configuration getShareLibConf(String inputKey, Path path) { - Configuration conf = new Configuration(); if (shareLibConfigMap.containsKey(inputKey)) { - conf = shareLibConfigMap.get(inputKey).get(path); + return shareLibConfigMap.get(inputKey).get(path); } - return conf; + return null; } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/oozie/blob/16234592/core/src/main/java/org/apache/oozie/util/JobUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/JobUtils.java b/core/src/main/java/org/apache/oozie/util/JobUtils.java index 6bb1a00..79edcd4 100644 --- a/core/src/main/java/org/apache/oozie/util/JobUtils.java +++ b/core/src/main/java/org/apache/oozie/util/JobUtils.java @@ -145,14 +145,11 @@ public class JobUtils { * @throws IOException */ public static void addFileToClassPath(Path file, Configuration conf, FileSystem fs) throws IOException { - Configuration defaultConf = new Configuration(); - XConfiguration.copy(conf, defaultConf); if (fs == null) { - // it fails with conf, therefore we pass defaultConf instead - fs = file.getFileSystem(defaultConf); + fs = file.getFileSystem(conf); } // Hadoop 0.20/1.x. - if (defaultConf.get("yarn.resourcemanager.webapp.address") == null) { + if (conf.get("yarn.resourcemanager.webapp.address") == null) { // Duplicate hadoop 1.x code to workaround MAPREDUCE-2361 in Hadoop 0.20 // Refer OOZIE-1806. String filepath = file.toUri().getPath(); http://git-wip-us.apache.org/repos/asf/oozie/blob/16234592/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 079d7b8..8b867bb 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -2247,7 +2247,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { // test .jar without fragment where app path is on same cluster as jar path Path appJarPath = new Path("lib/a.jar"); Path appJarFullPath = new Path(appPath, appJarPath); - conf.clear(); + conf = new Configuration(); conf.set(WorkflowAppService.HADOOP_USER, getTestUser()); ae.addToCache(conf, appPath, appJarFullPath.toString(), false); // assert that mapred.cache.files contains jar URI path (full on Hadoop-2) http://git-wip-us.apache.org/repos/asf/oozie/blob/16234592/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 7174312..b2cc29b 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2347 Remove unnecessary new Configuration()/new jobConf() calls from oozie (puru) OOZIE-2348 Recovery service keeps on recovering coord action of suspended jobs (puru) OOZIE-2277 Honor oozie.action.sharelib.for.spark in Spark jobs (rkanter) OOZIE-2322 Oozie Web UI doesn't work with Kerberos in Internet Explorer 10 or 11 and curl (rkanter)
