Repository: hadoop Updated Branches: refs/remotes/origin/branch-3.1 20453488e -> 6f31faf92
YARN-8372. Distributed shell app master should not release containers when shutdown if keep-container is true. (Suma Shivaprasad via wangda) Change-Id: Ief04d1ca865621f348fba4ac85fa78bc47465904 (cherry picked from commit 8956e5b8db3059e0872e49f59adc6affc76e2274) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed205764 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed205764 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed205764 Branch: refs/remotes/origin/branch-3.1 Commit: ed20576413c5ba6f2437d316d2904cfa9dda1958 Parents: 1cee85d Author: Wangda Tan <wan...@apache.org> Authored: Fri Jun 1 14:05:02 2018 -0700 Committer: Wangda Tan <wan...@apache.org> Committed: Fri Jun 1 15:47:59 2018 -0700 ---------------------------------------------------------------------- .../distributedshell/ApplicationMaster.java | 24 +++++++++++++++++--- .../applications/distributedshell/Client.java | 14 ++++++++---- 2 files changed, 31 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed205764/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index bc018b1..76fa38f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -269,6 +269,8 @@ public class ApplicationMaster { private String containerResourceProfile = ""; Map<String, Resource> resourceProfiles; + private boolean keepContainersAcrossAttempts = false; + // Counter for completed containers ( complete denotes successful or failed ) private AtomicInteger numCompletedContainers = new AtomicInteger(); // Allocated container count so that we know how many containers has the RM @@ -483,6 +485,13 @@ public class ApplicationMaster { + " the number of container retry attempts"); opts.addOption("placement_spec", true, "Placement specification"); opts.addOption("debug", false, "Dump out debug information"); + opts.addOption("keep_containers_across_application_attempts", false, + "Flag to indicate whether to keep containers across application " + + "attempts." + + " If the flag is true, running containers will not be killed when" + + " application attempt fails and these containers will be " + + "retrieved by" + + " the new application attempt "); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); @@ -646,6 +655,9 @@ public class ApplicationMaster { containerResourceProfile = cliParser.getOptionValue("container_resource_profile", ""); + keepContainersAcrossAttempts = cliParser.hasOption( + "keep_containers_across_application_attempts"); + if (this.placementSpecs == null) { numTotalContainers = Integer.parseInt(cliParser.getOptionValue( "num_containers", "1")); @@ -1152,9 +1164,15 @@ public class ApplicationMaster { } } - @Override - public void onShutdownRequest() { - done = true; + @Override public void onShutdownRequest() { + if (keepContainersAcrossAttempts) { + LOG.info("Shutdown request received. Ignoring since " + + "keep_containers_across_application_attempts is enabled"); + } else{ + LOG.info("Shutdown request received. Processing since " + + "keep_containers_across_application_attempts is disabled"); + done = true; + } } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed205764/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 61879d0..976e6a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -332,10 +332,12 @@ public class Client { + " containers to guaranteed."); opts.addOption("log_properties", true, "log4j.properties file"); opts.addOption("keep_containers_across_application_attempts", false, - "Flag to indicate whether to keep containers across application attempts." + - " If the flag is true, running containers will not be killed when" + - " application attempt fails and these containers will be retrieved by" + - " the new application attempt "); + "Flag to indicate whether to keep containers across application " + + "attempts." + + " If the flag is true, running containers will not be killed when" + + " application attempt fails and these containers will be " + + "retrieved by" + + " the new application attempt "); opts.addOption("attempt_failures_validity_interval", true, "when attempt_failures_validity_interval in milliseconds is set to > 0," + "the failure number will not take failures which happen out of " + @@ -891,6 +893,10 @@ public class Client { } vargs.add("--priority " + String.valueOf(shellCmdPriority)); + if (keepContainers) { + vargs.add("--keep_containers_across_application_attempts"); + } + for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
