Repository: oozie Updated Branches: refs/heads/master 9e8598ea4 -> 268a09f92
OOZIE-2869 Ability to add/remove MapReduce jars from action classpath (gezapeti) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/268a09f9 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/268a09f9 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/268a09f9 Branch: refs/heads/master Commit: 268a09f9243cd06106cc1795a97d0c1b52a8aee0 Parents: 9e8598e Author: Gezapeti Cseh <gezap...@apache.org> Authored: Fri Nov 17 14:00:33 2017 +0100 Committer: Gezapeti Cseh <gezap...@apache.org> Committed: Fri Nov 17 14:00:33 2017 +0100 ---------------------------------------------------------------------- .../action/hadoop/DistcpActionExecutor.java | 5 --- .../oozie/action/hadoop/HiveActionExecutor.java | 5 --- .../oozie/action/hadoop/JavaActionExecutor.java | 19 +++++---- .../action/hadoop/MapReduceActionExecutor.java | 6 --- .../oozie/action/hadoop/PigActionExecutor.java | 5 --- .../action/hadoop/SparkActionExecutor.java | 5 --- .../action/hadoop/SqoopActionExecutor.java | 5 --- core/src/main/resources/oozie-default.xml | 42 ++++++++++++++++++++ release-log.txt | 1 + sharelib/spark/pom.xml | 6 ++- 10 files changed, 60 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/268a09f9/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java index 961994e..81e28f7 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java @@ -107,11 +107,6 @@ public class DistcpActionExecutor extends JavaActionExecutor{ } @Override - protected boolean needToAddMapReduceToClassPath() { - return true; - } - - @Override protected String getLauncherMain(Configuration launcherConf, Element actionXml) { return launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS); } http://git-wip-us.apache.org/repos/asf/oozie/blob/268a09f9/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java index d8c613b..f07f431 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java @@ -132,11 +132,6 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor { } @Override - protected boolean needToAddMapReduceToClassPath() { - return true; - } - - @Override protected Configuration loadHadoopDefaultResources(Context context, Element actionXml) { boolean loadDefaultResources = ConfigurationService .getBoolean(HadoopAccessorService.ACTION_CONFS_LOAD_DEFAULT_RESOURCES); http://git-wip-us.apache.org/repos/asf/oozie/blob/268a09f9/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 8cb76cf..a1df304 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -136,6 +136,9 @@ public class JavaActionExecutor extends ActionExecutor { public static final String LAUNCER_MODIFY_ACL = "oozie.launcher.modify.acl"; public static final String LAUNCER_VIEW_ACL = "oozie.launcher.view.acl"; + public static final String MAPREDUCE_TO_CLASSPATH = "mapreduce.needed.for"; + public static final String OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY = ActionExecutor.CONF_PREFIX + MAPREDUCE_TO_CLASSPATH; + public static final String MAX_EXTERNAL_STATS_SIZE = "oozie.external.stats.max.size"; public static final String ACL_VIEW_JOB = "mapreduce.job.acl-view-job"; public static final String ACL_MODIFY_JOB = "mapreduce.job.acl-modify-job"; @@ -1253,20 +1256,20 @@ public class JavaActionExecutor extends ActionExecutor { appContext.setApplicationName(jobName); } - private void setEnvironmentVariables(Configuration launcherJobConf, ContainerLaunchContext amContainer) throws IOException { + private void setEnvironmentVariables(Configuration launcherConf, ContainerLaunchContext amContainer) throws IOException { Map<String, String> env = new HashMap<>(); - final String oozieLauncherEnvProperty = launcherJobConf.get(LauncherAM.OOZIE_LAUNCHER_ENV_PROPERTY); + final String oozieLauncherEnvProperty = launcherConf.get(LauncherAM.OOZIE_LAUNCHER_ENV_PROPERTY); if (oozieLauncherEnvProperty != null) { Map<String, String> environmentVars = extractEnvVarsFromOozieLauncherProps(oozieLauncherEnvProperty); env.putAll(environmentVars); } // This adds the Hadoop jars to the classpath in the Launcher JVM - ClasspathUtils.setupClasspath(env, launcherJobConf); + ClasspathUtils.setupClasspath(env, launcherConf); - if (needToAddMapReduceToClassPath()) { - ClasspathUtils.addMapReduceToClasspath(env, launcherJobConf); + if (needToAddMapReduceToClassPath(launcherConf)) { + ClasspathUtils.addMapReduceToClasspath(env, launcherConf); } addActionSpecificEnvVars(env); @@ -1540,9 +1543,11 @@ public class JavaActionExecutor extends ActionExecutor { * Subclasses should override this method if necessary. By default we don't add * MR jars to the classpath. * @return false by default + * @param launcherConf */ - protected boolean needToAddMapReduceToClassPath() { - return false; + private boolean needToAddMapReduceToClassPath(Configuration launcherConf) { + boolean defaultValue = launcherConf.getBoolean(OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY, false); + return launcherConf.getBoolean(OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY + "." + getType(), defaultValue); } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/268a09f9/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java index 233dce7..d0b8074 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java @@ -25,7 +25,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -339,11 +338,6 @@ public class MapReduceActionExecutor extends JavaActionExecutor { } @Override - protected boolean needToAddMapReduceToClassPath() { - return true; - } - - @Override public void check(Context context, WorkflowAction action) throws ActionExecutorException { Map<String, String> actionData = Collections.emptyMap(); Configuration jobConf = null; http://git-wip-us.apache.org/repos/asf/oozie/blob/268a09f9/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java index 6541d52..8465acd 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java @@ -123,11 +123,6 @@ public class PigActionExecutor extends ScriptLanguageActionExecutor { } @Override - protected boolean needToAddMapReduceToClassPath() { - return true; - } - - @Override protected Configuration loadHadoopDefaultResources(Context context, Element actionXml) { boolean loadDefaultResources = ConfigurationService .getBoolean(HadoopAccessorService.ACTION_CONFS_LOAD_DEFAULT_RESOURCES); http://git-wip-us.apache.org/repos/asf/oozie/blob/268a09f9/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index 80d64ec..2338ad2 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@ -141,11 +141,6 @@ public class SparkActionExecutor extends JavaActionExecutor { } @Override - protected boolean needToAddMapReduceToClassPath() { - return true; - } - - @Override protected void addActionSpecificEnvVars(Map<String, String> env) { env.put("SPARK_HOME", "."); } http://git-wip-us.apache.org/repos/asf/oozie/blob/268a09f9/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java index a0dfd31..69d5e7e 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java @@ -235,11 +235,6 @@ public class SqoopActionExecutor extends JavaActionExecutor { } } - @Override - protected boolean needToAddMapReduceToClassPath() { - return true; - } - /** * Return the sharelib name for the action. * http://git-wip-us.apache.org/repos/asf/oozie/blob/268a09f9/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 91d2811..1fd4ee3 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -3235,4 +3235,46 @@ will be the requeue interval for the actions which are waiting for a long time w The first one that is found will be used. </description> </property> + + <property> + <name>oozie.action.mapreduce.needed.for.distcp</name> + <value>true</value> + <description> + Whether to add MapReduce jars to the DistCp action's classpath's by default. + </description> + </property> + + <property> + <name>oozie.action.mapreduce.needed.for.hive</name> + <value>true</value> + <description> + Whether to add MapReduce jars to the Hive action's classpath's by default. + </description> + </property> + + <property> + <name>oozie.action.mapreduce.needed.for.map-reduce</name> + <value>true</value> + <description> + Whether to add MapReduce jars to the Map-Reduce action's classpath's by default. + </description> + </property> + + <property> + <name>oozie.action.mapreduce.needed.for.pig</name> + <value>true</value> + <description> + Whether to add MapReduce jars to the Pig action's classpath's by default. + </description> + </property> + + <property> + <name>oozie.action.mapreduce.needed.for.sqoop</name> + <value>true</value> + <description> + Whether to add MapReduce jars to the Sqoop action's classpath's by default. + </description> + </property> + + </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/268a09f9/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 8db0dde..04b04f3 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-2869 Ability to add/remove MapReduce jars from action classpath (gezapeti) OOZIE-2296 Add an Oozie diagnostic bundle tool (asasvari) OOZIE-3125 TestDBLoadDump.testImportInvalidDataLeavesTablesEmpty fails (asasvari) OOZIE-3106 upgrade surefire-plugin to 2.20.1 (dbist13 via asasvari) http://git-wip-us.apache.org/repos/asf/oozie/blob/268a09f9/sharelib/spark/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/spark/pom.xml b/sharelib/spark/pom.xml index b40a352..65f641a 100644 --- a/sharelib/spark/pom.xml +++ b/sharelib/spark/pom.xml @@ -43,7 +43,11 @@ <artifactId>guava</artifactId> <version>${spark.guava.version}</version> </dependency> - + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>compile</scope> + </dependency> <dependency> <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId>
