Repository: tez Updated Branches: refs/heads/master b04676932 -> cf400882b
TEZ-2627. Support for Tez Job Priorities. (Saikat via hitesh) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/cf400882 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/cf400882 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/cf400882 Branch: refs/heads/master Commit: cf400882bdc2247f3cf923d571c0863c6a556dd1 Parents: b046769 Author: Hitesh Shah <[email protected]> Authored: Mon Jul 27 15:48:14 2015 -0700 Committer: Hitesh Shah <[email protected]> Committed: Mon Jul 27 15:48:14 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/tez/client/TezClientUtils.java | 21 +++++++++++ .../apache/tez/dag/api/TezConfiguration.java | 6 ++++ .../apache/tez/client/TestTezClientUtils.java | 38 +++++++++++++++++++- .../java/org/apache/tez/test/TestTezJobs.java | 1 + 5 files changed, 66 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/cf400882/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 6d751ce..f7f2e76 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -11,6 +11,7 @@ INCOMPATIBLE CHANGES TEZ-2468. Change the minimum Java version to Java 7. ALL CHANGES: + TEZ-2627. Support for Tez Job Priorities. TEZ-2623. Fix module dependencies related to hadoop-auth. TEZ-2464. Move older releases to dist archive. TEZ-2239. Update Tez UI docs to explain how to configure history url for YARN. http://git-wip-us.apache.org/repos/asf/tez/blob/cf400882/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index 26d5e2a..8bfaa1f 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -662,6 +663,8 @@ public class TezClientUtils { if (amConfig.getQueueName() != null) { appContext.setQueue(amConfig.getQueueName()); } + // set the application priority + setApplicationPriority(appContext, amConfig); appContext.setApplicationName(amName); appContext.setCancelTokensWhenComplete(amConfig.getTezConfiguration().getBoolean( TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION, @@ -1008,4 +1011,22 @@ public class TezClientUtils { return null; } } + + @VisibleForTesting + // this method will set the app priority only if the priority config has been defined + public static void setApplicationPriority(ApplicationSubmissionContext context, + AMConfiguration amConfig) { + if (amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY) != null) { + // since we already checked not null condition, we are guaranteed that default value + // (0 in this case for getInt) will not be returned/used. + // The idea is to not use any default priority from TEZ side, if none provided in config; + // let YARN determine the priority of the submitted application + int priority = amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, 0); + context.setPriority(Priority.newInstance(priority)); + if (LOG.isDebugEnabled()) { + LOG.debug("Settting TEZ application priority, applicationId= " + context.getApplicationId() + + ", priority= " + context.getPriority().getPriority()); + } + } + } } http://git-wip-us.apache.org/repos/asf/tez/blob/cf400882/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 2114793..3b7378a 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -912,6 +912,12 @@ public class TezConfiguration extends Configuration { @ConfigurationScope(Scope.AM) public static final String TEZ_QUEUE_NAME = TEZ_PREFIX + "queue.name"; + /** + * String value. The job priority of submitted application. + */ + @ConfigurationScope(Scope.AM) + public static final String TEZ_AM_APPLICATION_PRIORITY = TEZ_PREFIX + "am.application.priority"; + @Unstable /** * Boolean value. Generate debug artifacts like DAG plan in text format. http://git-wip-us.apache.org/repos/asf/tez/blob/cf400882/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java index cfba917..2d4e005 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.Records; import org.apache.tez.common.security.HistoryACLPolicyManager; import org.apache.tez.common.security.JobTokenIdentifier; import org.apache.tez.common.security.JobTokenSecretManager; @@ -71,7 +72,6 @@ import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto; import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair; import org.junit.Assert; import org.junit.Test; - /** * */ @@ -204,6 +204,42 @@ public class TestTezClientUtils { assertFalse(localizedMap.isEmpty()); } + @Test(timeout = 2000) + // this test checks if the priority field is set properly in the + // ApplicationSubmissionContext + public void testAppSubmissionContextForPriority() throws Exception { + TezConfiguration tezConf = new TezConfiguration(); + int testpriority = 999; + ApplicationId appId = ApplicationId.newInstance(1000, 1); + Credentials credentials = new Credentials(); + TezClientUtils.createSessionToken(appId.toString(), + new JobTokenSecretManager(), credentials); + tezConf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true); + Map<String, LocalResource> m = new HashMap<String, LocalResource>(); + tezConf.setInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, testpriority); + AMConfiguration amConf = + new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials); + ApplicationSubmissionContext appcontext; + appcontext = TezClientUtils.createApplicationSubmissionContext( + appId, null, "dagname", + amConf, m, + credentials, false, + new TezApiVersionInfo(), null); + assertEquals(testpriority, appcontext.getPriority().getPriority()); + } + + @Test(timeout=1000) + // when tez config property for app priority not set, + // tez should not set app priority and let YARN deal with it. + public void testSetApplicationPriority() { + TezConfiguration conf = new TezConfiguration(false); + AMConfiguration amconfig = new AMConfiguration(conf, null, null); + ApplicationSubmissionContext appContext = Records + .newRecord(ApplicationSubmissionContext.class); + TezClientUtils.setApplicationPriority(appContext, amconfig); + assertNull(appContext.getPriority()); + } + @Test(timeout = 5000) public void testSessionTokenInAmClc() throws IOException, YarnException { http://git-wip-us.apache.org/repos/asf/tez/blob/cf400882/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java ---------------------------------------------------------------------- diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java index 0d0eaac..ab89ddb 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java @@ -299,6 +299,7 @@ public class TestTezJobs { String[] args = new String[] { "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "=" + stagingDirPath.toString(), + "-D" + TezConfiguration.TEZ_AM_APPLICATION_PRIORITY + "=" + "2", "-counter", inPath1.toString(), inPath2.toString(), "1", outPath.toString() }; assertEquals(0, sortMergeJoinExample.run(args));
