Repository: oozie Updated Branches: refs/heads/master 7739e93c0 -> 2545262f0
OOZIE-1722 When an ApplicationMaster restarts, it restarts the launcher job (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/2545262f Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/2545262f Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/2545262f Branch: refs/heads/master Commit: 2545262f037554849e3dd0069290d0b6aed37cb6 Parents: 7739e93 Author: Robert Kanter <[email protected]> Authored: Wed Mar 5 22:11:41 2014 -0800 Committer: Robert Kanter <[email protected]> Committed: Wed Mar 5 22:11:41 2014 -0800 ---------------------------------------------------------------------- .../action/hadoop/DistcpActionExecutor.java | 14 +-- .../oozie/action/hadoop/JavaActionExecutor.java | 25 ++++- .../action/hadoop/LauncherMapperHelper.java | 23 +++- .../action/hadoop/TestJavaActionExecutor.java | 9 +- .../oozie/action/hadoop/TestLauncher.java | 1 + hadooplibs/hadoop-utils-0.23/pom.xml | 43 ++++++++ .../action/hadoop/LauncherMainHadoopUtils.java | 35 ++++++ hadooplibs/hadoop-utils-1/pom.xml | 43 ++++++++ .../action/hadoop/LauncherMainHadoopUtils.java | 35 ++++++ hadooplibs/hadoop-utils-2/pom.xml | 43 ++++++++ .../action/hadoop/LauncherMainHadoopUtils.java | 35 ++++++ hadooplibs/hadoop-utils-3/pom.xml | 43 ++++++++ .../action/hadoop/LauncherMainHadoopUtils.java | 108 +++++++++++++++++++ hadooplibs/pom.xml | 4 + pom.xml | 6 ++ release-log.txt | 1 + .../action/hadoop/TestDistCpActionExecutor.java | 7 ++ sharelib/hive/pom.xml | 5 + .../apache/oozie/action/hadoop/HiveMain.java | 2 + sharelib/oozie/pom.xml | 5 + .../apache/oozie/action/hadoop/JavaMain.java | 84 +++++++++++++++ .../oozie/action/hadoop/LauncherMapper.java | 22 +++- .../oozie/action/hadoop/MapReduceMain.java | 32 ++++-- sharelib/pig/pom.xml | 5 + .../org/apache/oozie/action/hadoop/PigMain.java | 2 + sharelib/sqoop/pom.xml | 5 + .../apache/oozie/action/hadoop/SqoopMain.java | 2 + 27 files changed, 613 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java index fcf69e7..fe31d7b 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java @@ -20,6 +20,8 @@ package org.apache.oozie.action.hadoop; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.service.Services; import org.apache.oozie.util.XLog; import org.jdom.Element; @@ -35,22 +37,22 @@ public class DistcpActionExecutor extends JavaActionExecutor{ super("distcp"); } - /* (non-Javadoc) - * @see org.apache.oozie.action.hadoop.JavaActionExecutor#getLauncherMain(org.apache.hadoop.conf.Configuration, org.jdom.Element) - */ @Override - protected String getLauncherMain(Configuration launcherConf, Element actionXml) { + Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath) + throws ActionExecutorException { + actionConf = super.setupActionConf(actionConf, context, actionXml, appPath); String classNameDistcp = CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS; String name = getClassNamebyType(DISTCP_TYPE); if(name != null){ classNameDistcp = name; } - return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, classNameDistcp); + actionConf.set(JavaMain.JAVA_MAIN_CLASS, classNameDistcp); + return actionConf; } @Override public List<Class> getLauncherClasses() { - return null; + return super.getLauncherClasses(); } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 68d77a8..860eb51 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -101,6 +101,7 @@ public class JavaActionExecutor extends ActionExecutor { public static final String YARN_AM_RESOURCE_MB = "yarn.app.mapreduce.am.resource.mb"; public static final String YARN_AM_COMMAND_OPTS = "yarn.app.mapreduce.am.command-opts"; public static final String YARN_AM_ENV = "yarn.app.mapreduce.am.env"; + private static final String JAVA_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.JavaMain"; public static final int YARN_MEMORY_MB_MIN = 512; private static int maxActionOutputLen; private static int maxExternalStatsSize; @@ -138,7 +139,14 @@ public class JavaActionExecutor extends ActionExecutor { } public List<Class> getLauncherClasses() { - return null; + List<Class> classes = new ArrayList<Class>(); + try { + classes.add(Class.forName(JAVA_MAIN_CLASS_NAME)); + } + catch (ClassNotFoundException e) { + throw new RuntimeException("Class not found", e); + } + return classes; } @Override @@ -386,6 +394,9 @@ public class JavaActionExecutor extends ActionExecutor { has.checkSupportedFilesystem(appPath.toUri()); + // Set the Java Main Class for the Java action to give to the Java launcher + setJavaMain(actionConf, actionXml); + parseJobXmlAndConfiguration(context, actionXml, appPath, actionConf); return actionConf; } @@ -657,9 +668,15 @@ public class JavaActionExecutor extends ActionExecutor { protected String getLauncherMain(Configuration launcherConf, Element actionXml) { + return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, JavaMain.class.getName()); + } + + private void setJavaMain(Configuration actionConf, Element actionXml) { Namespace ns = actionXml.getNamespace(); Element e = actionXml.getChild("main-class", ns); - return e.getTextTrim(); + if (e != null) { + actionConf.set(JavaMain.JAVA_MAIN_CLASS, e.getTextTrim()); + } } private static final String QUEUE_NAME = "mapred.job.queue.name"; @@ -687,6 +704,9 @@ public class JavaActionExecutor extends ActionExecutor { JobConf launcherJobConf = createBaseHadoopConf(context, actionXml); setupLauncherConf(launcherJobConf, actionXml, appPathRoot, context); + // Properties for when a launcher job's AM gets restarted + LauncherMapperHelper.setupYarnRestartHandling(launcherJobConf, actionConf, action.getId()); + String actionShareLibProperty = actionConf.get(ACTION_SHARELIB_FOR + getType()); if (actionShareLibProperty != null) { launcherJobConf.set(ACTION_SHARELIB_FOR + getType(), actionShareLibProperty); @@ -719,6 +739,7 @@ public class JavaActionExecutor extends ActionExecutor { LauncherMapperHelper.setupLauncherInfo(launcherJobConf, jobId, actionId, actionDir, recoveryId, actionConf, prepareXML); + // Set the launcher Main Class LauncherMapperHelper.setupMainClass(launcherJobConf, getLauncherMain(launcherJobConf, actionXml)); LauncherMapperHelper.setupLauncherURIHandlerConf(launcherJobConf); LauncherMapperHelper.setupMaxOutputData(launcherJobConf, maxActionOutputLen); http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java index 07de214..eeda715 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java @@ -24,6 +24,9 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collection; @@ -42,6 +45,7 @@ import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.security.UserGroupInformation; import org.apache.oozie.client.OozieClient; +import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.Services; @@ -70,8 +74,8 @@ public class LauncherMapperHelper { } public static void setupMainClass(Configuration launcherConf, String javaMainClass) { - // Only set the javaMainClass if its not null or empty string (should be the case except for java action), this way the user - // can override the action's main class via <configuration> property + // Only set the javaMainClass if its not null or empty string, this way the user can override the action's main class via + // <configuration> property if (javaMainClass != null && !javaMainClass.equals("")) { launcherConf.set(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, javaMainClass); } @@ -162,6 +166,21 @@ public class LauncherMapperHelper { launcherConf.set("mapred.output.dir", new Path(actionDir, "output").toString()); } + public static void setupYarnRestartHandling(JobConf launcherJobConf, Configuration actionConf, String actionId) + throws NoSuchAlgorithmException { + launcherJobConf.setLong("oozie.job.launch.time", System.currentTimeMillis()); + // Tags are limited to 100 chars so we need to hash them to make sure (the actionId otherwise doesn't have a max length) + String tag = getTag(actionId); + actionConf.set("mapreduce.job.tags", tag); + } + + private static String getTag(String actionId) throws NoSuchAlgorithmException { + MessageDigest digest = MessageDigest.getInstance("MD5"); + digest.update(actionId.getBytes(), 0, actionId.length()); + String md5 = "oozie-" + new BigInteger(1, digest.digest()).toString(16); + return md5; + } + public static boolean isMainDone(RunningJob runningJob) throws IOException { return runningJob.isComplete(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 7841076..c8b8caf 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -25,11 +25,9 @@ import java.io.OutputStream; import java.io.StringReader; import java.net.URI; import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; -import java.util.List; import java.util.Properties; import org.apache.hadoop.conf.Configuration; @@ -57,7 +55,6 @@ import org.apache.oozie.service.ShareLibService; import org.apache.oozie.service.UUIDService; import org.apache.oozie.service.WorkflowAppService; import org.apache.oozie.service.WorkflowStoreService; -import org.apache.oozie.util.HCatURI; import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XmlUtils; @@ -88,9 +85,10 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { } + @SuppressWarnings("unchecked") public void testSetupMethods() throws Exception { JavaActionExecutor ae = new JavaActionExecutor(); - assertEquals(null, ae.getLauncherClasses()); + assertEquals(Arrays.asList(JavaMain.class), ae.getLauncherClasses()); Configuration conf = new XConfiguration(); conf.set("user.name", "a"); try { @@ -241,7 +239,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { conf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf); ae.setupLauncherConf(conf, actionXml, getFsTestCaseDir(), context); - assertEquals("MAIN-CLASS", ae.getLauncherMain(conf, actionXml)); + assertEquals("MAIN-CLASS", actionConf.get("oozie.action.java.main", "null")); + assertEquals("org.apache.oozie.action.hadoop.JavaMain", ae.getLauncherMain(conf, actionXml)); assertTrue(conf.get("mapred.child.java.opts").contains("JAVA-OPTS")); assertEquals(Arrays.asList("A1", "A2"), Arrays.asList(LauncherMapper.getMainArguments(conf))); http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java index b55b9d7..67eda7e 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java @@ -78,6 +78,7 @@ public class TestLauncher extends XFsTestCase { Configuration actionConf = new XConfiguration(); LauncherMapperHelper.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, ""); + LauncherMapperHelper.setupYarnRestartHandling(jobConf, jobConf, "1@a"); assertEquals("1", actionConf.get("oozie.job.id")); assertEquals("1@a", actionConf.get("oozie.action.id")); http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-0.23/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-0.23/pom.xml b/hadooplibs/hadoop-utils-0.23/pom.xml new file mode 100644 index 0000000..13e736f --- /dev/null +++ b/hadooplibs/hadoop-utils-0.23/pom.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-main</artifactId> + <version>4.1.0-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop-utils</artifactId> + <version>0.23.5.oozie-4.1.0-SNAPSHOT</version> + <description>Apache Oozie Hadoop Utils ${project.version}</description> + <name>Apache Oozie Hadoop Utils ${project.version}</name> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java new file mode 100644 index 0000000..07a3c29 --- /dev/null +++ b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.action.hadoop; + +import org.apache.hadoop.conf.Configuration; + + +public class LauncherMainHadoopUtils { + + private LauncherMainHadoopUtils() { + } + + public static String getYarnJobForMapReduceAction(Configuration actionConf) { + return null; + } + + public static void killChildYarnJobs(Configuration actionConf) { + // no-op + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-1/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-1/pom.xml b/hadooplibs/hadoop-utils-1/pom.xml new file mode 100644 index 0000000..0d10e7a --- /dev/null +++ b/hadooplibs/hadoop-utils-1/pom.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-main</artifactId> + <version>4.1.0-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop-utils</artifactId> + <version>1.1.1.oozie-4.1.0-SNAPSHOT</version> + <description>Apache Oozie Hadoop Utils ${project.version}</description> + <name>Apache Oozie Hadoop Utils ${project.version}</name> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java new file mode 100644 index 0000000..07a3c29 --- /dev/null +++ b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.action.hadoop; + +import org.apache.hadoop.conf.Configuration; + + +public class LauncherMainHadoopUtils { + + private LauncherMainHadoopUtils() { + } + + public static String getYarnJobForMapReduceAction(Configuration actionConf) { + return null; + } + + public static void killChildYarnJobs(Configuration actionConf) { + // no-op + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-2/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-2/pom.xml b/hadooplibs/hadoop-utils-2/pom.xml new file mode 100644 index 0000000..3f48066 --- /dev/null +++ b/hadooplibs/hadoop-utils-2/pom.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-main</artifactId> + <version>4.1.0-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop-utils</artifactId> + <version>2.2.0.oozie-4.1.0-SNAPSHOT</version> + <description>Apache Oozie Hadoop Utils ${project.version}</description> + <name>Apache Oozie Hadoop Utils ${project.version}</name> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java new file mode 100644 index 0000000..07a3c29 --- /dev/null +++ b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.action.hadoop; + +import org.apache.hadoop.conf.Configuration; + + +public class LauncherMainHadoopUtils { + + private LauncherMainHadoopUtils() { + } + + public static String getYarnJobForMapReduceAction(Configuration actionConf) { + return null; + } + + public static void killChildYarnJobs(Configuration actionConf) { + // no-op + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-3/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-3/pom.xml b/hadooplibs/hadoop-utils-3/pom.xml new file mode 100644 index 0000000..07afb2d --- /dev/null +++ b/hadooplibs/hadoop-utils-3/pom.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-main</artifactId> + <version>4.1.0-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop-utils</artifactId> + <version>3.0.0-SNAPSHOT.oozie-4.1.0-SNAPSHOT</version> + <description>Apache Oozie Hadoop Utils ${project.version}</description> + <name>Apache Oozie Hadoop Utils ${project.version}</name> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java new file mode 100644 index 0000000..db827e5 --- /dev/null +++ b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.action.hadoop; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.mapreduce.TypeConverter; + +public class LauncherMainHadoopUtils { + + private LauncherMainHadoopUtils() { + } + + private static Set<ApplicationId> getChildYarnJobs(Configuration actionConf) { + Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>(); + long startTime = 0L; + try { + startTime = Long.parseLong((System.getProperty("oozie.job.launch.time"))); + } catch(NumberFormatException nfe) { + throw new RuntimeException("Could not find Oozie job launch time", nfe); + } + String tag = actionConf.get("mapreduce.job.tags"); + if (tag == null) { + throw new RuntimeException("Could not find Yarn tags property (mapreduce.job.tags)"); + } + GetApplicationsRequest gar = GetApplicationsRequest.newInstance(); + gar.setScope(ApplicationsRequestScope.OWN); + gar.setStartRange(startTime, System.currentTimeMillis()); + gar.setApplicationTags(Collections.singleton(tag)); + try { + ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(actionConf, ApplicationClientProtocol.class); + GetApplicationsResponse apps = proxy.getApplications(gar); + List<ApplicationReport> appsList = apps.getApplicationList(); + for(ApplicationReport appReport : appsList) { + childYarnJobs.add(appReport.getApplicationId()); + } + } catch (IOException ioe) { + throw new RuntimeException("Exception occurred while finding child jobs", ioe); + } catch (YarnException ye) { + throw new RuntimeException("Exception occurred while finding child jobs", ye); + } + return childYarnJobs; + } + + public static String getYarnJobForMapReduceAction(Configuration actionConf) { + Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf); + String childJobId = null; + if (!childYarnJobs.isEmpty()) { + ApplicationId childJobYarnId = childYarnJobs.iterator().next(); + System.out.println("Found Map-Reduce job [" + childJobYarnId + "] already running"); + // Need the JobID version for Oozie + childJobId = TypeConverter.fromYarn(childJobYarnId).toString(); + } + return childJobId; + } + + public static void killChildYarnJobs(Configuration actionConf) { + try { + Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf); + if (!childYarnJobs.isEmpty()) { + System.out.println(); + System.out.println("Found [" + childYarnJobs.size() + "] Map-Reduce jobs from this launcher"); + System.out.println("Killing existing jobs and starting over:"); + YarnClient yarnClient = YarnClient.createYarnClient(); + yarnClient.init(actionConf); + yarnClient.start(); + for (ApplicationId app : childYarnJobs) { + System.out.print("Killing job [" + app + "] ... "); + yarnClient.killApplication(app); + System.out.println("Done"); + } + System.out.println(); + } + } catch (YarnException ye) { + throw new RuntimeException("Exception occurred while killing child job(s)", ye); + } catch (IOException ioe) { + throw new RuntimeException("Exception occurred while killing child job(s)", ioe); + } + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/pom.xml b/hadooplibs/pom.xml index 35875c8..788788c 100644 --- a/hadooplibs/pom.xml +++ b/hadooplibs/pom.xml @@ -35,12 +35,15 @@ <module>hadoop-1</module> <module>hadoop-distcp-1</module> <module>hadoop-test-1</module> + <module>hadoop-utils-1</module> <module>hadoop-2</module> <module>hadoop-test-2</module> <module>hadoop-distcp-2</module> + <module>hadoop-utils-2</module> <module>hadoop-0.23</module> <module>hadoop-test-0.23</module> <module>hadoop-distcp-0.23</module> + <module>hadoop-utils-0.23</module> </modules> <build> @@ -75,6 +78,7 @@ <module>hadoop-3</module> <module>hadoop-test-3</module> <module>hadoop-distcp-3</module> + <module>hadoop-utils-3</module> </modules> </profile> </profiles> http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index f960117..99fb083 100644 --- a/pom.xml +++ b/pom.xml @@ -780,6 +780,12 @@ <version>4.01</version> </dependency> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop-utils</artifactId> + <version>${hadooplib.version}</version> + </dependency> + </dependencies> </dependencyManagement> http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index adf1fef..51d8ada 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1722 When an ApplicationMaster restarts, it restarts the launcher job (rkanter) OOZIE-1727 Upgrade hadoop-minikdc (puru via rkanter) OOZIE-1711 TestLiteWorkflowAppParser fails against Hadoop 2 (rkanter) OOZIE-1714 Update Derby driver version to latest (rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/distcp/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/distcp/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java b/sharelib/distcp/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java index 25b1a23..b075957 100644 --- a/sharelib/distcp/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java +++ b/sharelib/distcp/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.InputStream; import java.io.OutputStream; +import java.util.Arrays; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobClient; @@ -44,6 +45,12 @@ public class TestDistCpActionExecutor extends ActionExecutorTestCase{ setSystemProperty("oozie.service.ActionService.executor.classes", DistcpActionExecutor.class.getName()); } + @SuppressWarnings("unchecked") + public void testSetupMethods() throws Exception { + DistcpActionExecutor ae = new DistcpActionExecutor(); + assertEquals(Arrays.asList(JavaMain.class), ae.getLauncherClasses()); + } + public void testDistCpFile() throws Exception { Path inputPath = new Path(getFsTestCaseDir(), "input.txt"); final Path outputPath = new Path(getFsTestCaseDir(), "output.txt"); http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/hive/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/hive/pom.xml b/sharelib/hive/pom.xml index e355d97..5ca5cb8 100644 --- a/sharelib/hive/pom.xml +++ b/sharelib/hive/pom.xml @@ -152,6 +152,11 @@ <artifactId>oozie-sharelib-oozie</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop-utils</artifactId> + <scope>provided</scope> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java index 8fa5ec8..348c606 100644 --- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java +++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java @@ -264,6 +264,8 @@ public class HiveMain extends LauncherMain { } System.out.println(); + LauncherMainHadoopUtils.killChildYarnJobs(hiveConf); + System.out.println("================================================================="); System.out.println(); System.out.println(">>> Invoking Hive command line now >>>"); http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/oozie/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/oozie/pom.xml b/sharelib/oozie/pom.xml index c70582d..22a01b6 100644 --- a/sharelib/oozie/pom.xml +++ b/sharelib/oozie/pom.xml @@ -58,6 +58,11 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop-utils</artifactId> + <scope>compile</scope> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java new file mode 100644 index 0000000..d7eb17b --- /dev/null +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +public class JavaMain extends LauncherMain { + public static final String JAVA_MAIN_CLASS = "oozie.action.java.main"; + + /** + * @param args Invoked from LauncherMapper:map() + * @throws Exception + */ + public static void main(String[] args) throws Exception { + run(JavaMain.class, args); + } + + @Override + protected void run(String[] args) throws Exception { + + Configuration actionConf = loadActionConf(); + + LauncherMainHadoopUtils.killChildYarnJobs(actionConf); + + Class<?> klass = actionConf.getClass(JAVA_MAIN_CLASS, Object.class); + System.out.println("Main class : " + klass.getName()); + System.out.println("Arguments :"); + for (String arg : args) { + System.out.println(" " + arg); + } + System.out.println(); + Method mainMethod = klass.getMethod("main", String[].class); + try { + mainMethod.invoke(null, (Object) args); + } catch(InvocationTargetException ex) { + // Get rid of the InvocationTargetException and wrap the Throwable + throw new JavaMainException(ex.getCause()); + } + } + + /** + * Read action configuration passes through action xml file. + * + * @return action Configuration + * @throws IOException + */ + protected Configuration loadActionConf() throws IOException { + // loading action conf prepared by Oozie + Configuration actionConf = new Configuration(false); + + String actionXml = System.getProperty("oozie.action.conf.xml"); + + if (actionXml == null) { + throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]"); + } + if (!new File(actionXml).exists()) { + throw new RuntimeException("Action Configuration XML file [" + actionXml + "] does not exist"); + } + + actionConf.addResource(new Path("file:///", actionXml)); + return actionConf; + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java index b5fa4a1..322f498 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java @@ -226,12 +226,18 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R mainMethod.invoke(null, (Object) args); } catch (InvocationTargetException ex) { - if (LauncherMainException.class.isInstance(ex.getCause())) { + // Get what actually caused the exception + Throwable cause = ex.getCause(); + // If we got a JavaMainException from JavaMain, then we need to unwrap it + if (JavaMainException.class.isInstance(cause)) { + cause = cause.getCause(); + } + if (LauncherMainException.class.isInstance(cause)) { errorMessage = msgPrefix + "exit code [" +((LauncherMainException)ex.getCause()).getErrorCode() + "]"; errorCause = null; } - else if (SecurityException.class.isInstance(ex.getCause())) { + else if (SecurityException.class.isInstance(cause)) { if (LauncherSecurityManager.getExitInvoked()) { System.out.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")"); @@ -426,7 +432,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R System.setProperty(ACTION_PREFIX + ACTION_DATA_NEW_ID, new File(ACTION_DATA_NEW_ID).getAbsolutePath()); System.setProperty(ACTION_PREFIX + ACTION_DATA_OUTPUT_PROPS, new File(ACTION_DATA_OUTPUT_PROPS).getAbsolutePath()); System.setProperty(ACTION_PREFIX + ACTION_DATA_ERROR_PROPS, new File(ACTION_DATA_ERROR_PROPS).getAbsolutePath()); - + System.setProperty("oozie.job.launch.time", getJobConf().get("oozie.job.launch.time")); } // Method to execute the prepare actions @@ -598,3 +604,13 @@ class LauncherSecurityManager extends SecurityManager { exitCode = 0; } } + +/** + * Used by JavaMain to wrap a Throwable when an Exception occurs + */ +@SuppressWarnings("serial") +class JavaMainException extends Exception { + public JavaMainException(Throwable t) { + super(t); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java index 6230e64..f417a78 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java @@ -48,20 +48,36 @@ public class MapReduceMain extends LauncherMain { logMasking("Map-Reduce job configuration:", new HashSet<String>(), actionConf); - System.out.println("Submitting Oozie action Map-Reduce job"); + String jobId = LauncherMainHadoopUtils.getYarnJobForMapReduceAction(actionConf); + File idFile = new File(System.getProperty(LauncherMapper.ACTION_PREFIX + LauncherMapper.ACTION_DATA_NEW_ID)); + if (jobId != null) { + if (!idFile.exists()) { + System.out.print("JobId file is mising: writing now... "); + writeJobIdFile(idFile, jobId); + System.out.print("Done"); + } + System.out.println("Exiting launcher"); + System.out.println(); + } + else { + System.out.println("Submitting Oozie action Map-Reduce job"); + System.out.println(); + // submitting job + RunningJob runningJob = submitJob(actionConf); + + jobId = runningJob.getID().toString(); + writeJobIdFile(idFile, jobId); + } + + System.out.println("======================="); System.out.println(); - // submitting job - RunningJob runningJob = submitJob(actionConf); + } + protected void writeJobIdFile(File idFile, String jobId) throws IOException { // propagating job id back to Oozie - String jobId = runningJob.getID().toString(); - File idFile = new File(System.getProperty(LauncherMapper.ACTION_PREFIX + LauncherMapper.ACTION_DATA_NEW_ID)); OutputStream os = new FileOutputStream(idFile); os.write(jobId.getBytes()); os.close(); - - System.out.println("======================="); - System.out.println(); } protected void addActionConf(JobConf jobConf, Configuration actionConf) { http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/pig/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/pig/pom.xml b/sharelib/pig/pom.xml index a2bbeaf..e05fefc 100644 --- a/sharelib/pig/pom.xml +++ b/sharelib/pig/pom.xml @@ -109,6 +109,11 @@ <artifactId>oozie-sharelib-oozie</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop-utils</artifactId> + <scope>provided</scope> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java index 9c1e659..7b16a8c 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java @@ -211,6 +211,8 @@ public class PigMain extends LauncherMain { System.out.println(" " + arg); } + LauncherMainHadoopUtils.killChildYarnJobs(actionConf); + System.out.println("================================================================="); System.out.println(); System.out.println(">>> Invoking Pig command line now >>>"); http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/sqoop/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/pom.xml b/sharelib/sqoop/pom.xml index a962565..8925376 100644 --- a/sharelib/sqoop/pom.xml +++ b/sharelib/sqoop/pom.xml @@ -217,6 +217,11 @@ <artifactId>oozie-sharelib-oozie</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-hadoop-utils</artifactId> + <scope>provided</scope> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java index e45c8b2..0194dcc 100644 --- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java +++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java @@ -162,6 +162,8 @@ public class SqoopMain extends LauncherMain { System.out.println(" " + arg); } + LauncherMainHadoopUtils.killChildYarnJobs(sqoopConf); + System.out.println("================================================================="); System.out.println(); System.out.println(">>> Invoking Sqoop command line now >>>");
