YARN-8798. [Submarine] Job should not be submitted if --input_path option is missing. (Zhankun Tang via wangda)
Change-Id: I7ae0e44eb5179b04a6ac861ec1c65f3b18c38f0f (cherry picked from commit 143d74775b2b62884090fdd88874134b9eab2888) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa23c8d7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa23c8d7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa23c8d7 Branch: refs/heads/branch-3.2 Commit: aa23c8d7396430f3863d56e2ea0d94ac3fdb34a0 Parents: 920401b Author: Wangda Tan <[email protected]> Authored: Tue Oct 16 13:39:34 2018 -0700 Committer: Wangda Tan <[email protected]> Committed: Tue Oct 16 13:53:03 2018 -0700 ---------------------------------------------------------------------- .../client/cli/param/RunJobParameters.java | 6 +++ .../client/cli/TestRunJobCliParsing.java | 39 ++++++++++++++++++++ 2 files changed, 45 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa23c8d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java index 92a1883..d923e0f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java @@ -62,6 +62,12 @@ public class RunJobParameters extends RunParameters { if (parsedCommandLine.getOptionValue(CliConstants.N_WORKERS) != null) { nWorkers = Integer.parseInt( parsedCommandLine.getOptionValue(CliConstants.N_WORKERS)); + // Only check null value. + // Training job shouldn't ignore INPUT_PATH option + // But if nWorkers is 0, INPUT_PATH can be ignored because user can only run Tensorboard + if (null == input && 0 != nWorkers) { + throw new ParseException("\"--" + CliConstants.INPUT_PATH + "\" is absent"); + } } int nPS = 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa23c8d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java index 295d6a8..240de06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.submarine.client.cli; +import org.apache.commons.cli.ParseException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; @@ -136,6 +137,44 @@ public class TestRunJobCliParsing { } @Test + public void testNoInputPathOptionSpecified() throws Exception { + RunJobCli runJobCli = new RunJobCli(getMockClientContext()); + String expectedErrorMessage = "\"--" + CliConstants.INPUT_PATH + "\" is absent"; + String actualMessage = ""; + try { + runJobCli.run( + new String[]{"--name", "my-job", "--docker_image", "tf-docker:1.1.0", + "--checkpoint_path", "hdfs://output", + "--num_workers", "1", "--worker_launch_cmd", "python run-job.py", + "--worker_resources", "memory=4g,vcores=2", "--tensorboard", + "true", "--verbose", "--wait_job_finish"}); + } catch (ParseException e) { + actualMessage = e.getMessage(); + e.printStackTrace(); + } + Assert.assertEquals(expectedErrorMessage, actualMessage); + } + + /** + * when only run tensorboard, input_path is not needed + * */ + @Test + public void testNoInputPathOptionButOnlyRunTensorboard() throws Exception { + RunJobCli runJobCli = new RunJobCli(getMockClientContext()); + boolean success = true; + try { + runJobCli.run( + new String[]{"--name", "my-job", "--docker_image", "tf-docker:1.1.0", + "--num_workers", "0", "--tensorboard", "--verbose", + "--tensorboard_resources", "memory=2G,vcores=2", + "--tensorboard_docker_image", "tb_docker_image:001"}); + } catch (ParseException e) { + success = false; + } + Assert.assertTrue(success); + } + + @Test public void testLaunchCommandPatternReplace() throws Exception { RunJobCli runJobCli = new RunJobCli(getMockClientContext()); Assert.assertFalse(SubmarineLogs.isVerbose()); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
