Repository: oozie Updated Branches: refs/heads/master d5f88032a -> bdd4914fa
OOZIE-1722 Add support to Hadoop-2 for AM restarts of the launcher job (jaydeepvishwakarma via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/bdd4914f Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/bdd4914f Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/bdd4914f Branch: refs/heads/master Commit: bdd4914fa95808bcc4192b5ccc8464554153ecd7 Parents: d5f8803 Author: Robert Kanter <[email protected]> Authored: Fri Jan 23 14:13:02 2015 -0800 Committer: Robert Kanter <[email protected]> Committed: Fri Jan 23 14:13:02 2015 -0800 ---------------------------------------------------------------------- .../action/hadoop/LauncherMainHadoopUtils.java | 79 +++++++++++++++++++- release-log.txt | 1 + 2 files changed, 77 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/bdd4914f/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java index 46c2fbd..f6bb6a4 100644 --- a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java +++ b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java @@ -18,19 +18,92 @@ package org.apache.oozie.action.hadoop; +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import org.apache.hadoop.conf.Configuration; - +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.mapreduce.TypeConverter; public class LauncherMainHadoopUtils { private LauncherMainHadoopUtils() { } + private static Set<ApplicationId> getChildYarnJobs(Configuration actionConf) { + Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>(); + long startTime = 0L; + try { + startTime = Long.parseLong((System.getProperty("oozie.job.launch.time"))); + } catch(NumberFormatException nfe) { + throw new RuntimeException("Could not find Oozie job launch time", nfe); + } + String tag = actionConf.get("mapreduce.job.tags"); + if (tag == null) { + throw new RuntimeException("Could not find Yarn tags property (mapreduce.job.tags)"); + } + GetApplicationsRequest gar = GetApplicationsRequest.newInstance(); + gar.setScope(ApplicationsRequestScope.OWN); + gar.setStartRange(startTime, System.currentTimeMillis()); + gar.setApplicationTags(Collections.singleton(tag)); + try { + ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(actionConf, ApplicationClientProtocol.class); + GetApplicationsResponse apps = proxy.getApplications(gar); + List<ApplicationReport> appsList = apps.getApplicationList(); + for(ApplicationReport appReport : appsList) { + childYarnJobs.add(appReport.getApplicationId()); + } + } catch (IOException ioe) { + throw new RuntimeException("Exception occurred while finding child jobs", ioe); + } catch (YarnException ye) { + throw new RuntimeException("Exception occurred while finding child jobs", ye); + } + return childYarnJobs; + } + public static String getYarnJobForMapReduceAction(Configuration actionConf) { - return null; + Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf); + String childJobId = null; + if (!childYarnJobs.isEmpty()) { + ApplicationId childJobYarnId = childYarnJobs.iterator().next(); + System.out.println("Found Map-Reduce job [" + childJobYarnId + "] already running"); + // Need the JobID version for Oozie + childJobId = TypeConverter.fromYarn(childJobYarnId).toString(); + } + return childJobId; } public static void killChildYarnJobs(Configuration actionConf) { - // no-op + try { + Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf); + if (!childYarnJobs.isEmpty()) { + System.out.println(); + System.out.println("Found [" + childYarnJobs.size() + "] Map-Reduce jobs from this launcher"); + System.out.println("Killing existing jobs and starting over:"); + YarnClient yarnClient = YarnClient.createYarnClient(); + yarnClient.init(actionConf); + yarnClient.start(); + for (ApplicationId app : childYarnJobs) { + System.out.print("Killing job [" + app + "] ... "); + yarnClient.killApplication(app); + System.out.println("Done"); + } + System.out.println(); + } + } catch (YarnException ye) { + throw new RuntimeException("Exception occurred while killing child job(s)", ye); + } catch (IOException ioe) { + throw new RuntimeException("Exception occurred while killing child job(s)", ioe); + } } } http://git-wip-us.apache.org/repos/asf/oozie/blob/bdd4914f/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 6dda7b1..b834cbc 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-1722 Add support to Hadoop-2 for AM restarts of the launcher job (jaydeepvishwakarma via rkanter) OOZIE-2107 Schema config properties should be consistent with ActionExecutor config properties (rkanter) OOZIE-1730 Change hadoop-2 profile to use 2.4.0 (jaydeepvishwakarma via rkanter) OOZIE-2088 Exponential retries for transient failures (pavan kumar via shwethags)
