Repository: tez Updated Branches: refs/heads/master 89802b1c5 -> de7fd9aa5
TEZ-909. Provide support for application tags (ebadger via rohini) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/de7fd9aa Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/de7fd9aa Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/de7fd9aa Branch: refs/heads/master Commit: de7fd9aa5dea40a3cd62fd8bee3c2bf96065a9fe Parents: 89802b1 Author: Rohini Palaniswamy <[email protected]> Authored: Fri May 27 12:32:21 2016 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Fri May 27 12:32:21 2016 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../java/org/apache/tez/client/TezClientUtils.java | 7 +++++++ .../org/apache/tez/dag/api/TezConfiguration.java | 8 ++++++++ .../org/apache/tez/client/TestTezClientUtils.java | 17 +++++++++++++++++ .../tez/mapreduce/hadoop/DeprecatedKeys.java | 2 ++ .../apache/tez/mapreduce/hadoop/MRJobConfig.java | 2 ++ 6 files changed, 37 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/de7fd9aa/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 07052e5..ed41b07 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-909. Provide support for application tags TEZ-3257. Fix flaky test TestUnorderedPartitionedKVWriter. TEZ-3206. Have unordered partitioned KV output send partition stats via VertexManagerEvent. TEZ-3237. Corrupted shuffle transfers to disk are not detected during transfer http://git-wip-us.apache.org/repos/asf/tez/blob/de7fd9aa/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index a1ad1d8..eb1a95e 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -704,7 +704,14 @@ public class TezClientUtils { ApplicationSubmissionContext appContext = Records .newRecord(ApplicationSubmissionContext.class); + Collection<String> tagsFromConf = + amConfig.getTezConfiguration().getTrimmedStringCollection( + TezConfiguration.TEZ_APPLICATION_TAGS); + appContext.setApplicationType(TezConstants.TEZ_APPLICATION_TYPE); + if (tagsFromConf != null && !tagsFromConf.isEmpty()) { + appContext.setApplicationTags(new HashSet<String>(tagsFromConf)); + } appContext.setApplicationId(appId); appContext.setResource(capability); if (amConfig.getQueueName() != null) { http://git-wip-us.apache.org/repos/asf/tez/blob/de7fd9aa/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 9f9defe..4118bb5 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -1141,6 +1141,14 @@ public class TezConfiguration extends Configuration { public static final String TEZ_QUEUE_NAME = TEZ_PREFIX + "queue.name"; /** + * String value. Tags for the job that will be passed to YARN at submission + * time. Queries to YARN for applications can filter on these tags. + */ + @ConfigurationScope(Scope.AM) + @ConfigurationProperty + public static final String TEZ_APPLICATION_TAGS = TEZ_PREFIX + "application.tags"; + + /** * String value. The job priority of submitted application. */ @ConfigurationScope(Scope.AM) http://git-wip-us.apache.org/repos/asf/tez/blob/de7fd9aa/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java index 4948260..2c69d77 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java @@ -32,7 +32,9 @@ import java.net.URLClassLoader; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -328,6 +330,21 @@ public class TestTezClientUtils { assertNull(appContext.getPriority()); } + @Test(timeout=1000) + public void testSetApplicationTags() { + TezConfiguration conf = new TezConfiguration(false); + conf.set(TezConfiguration.TEZ_APPLICATION_TAGS, "foo,bar"); + AMConfiguration amconfig = new AMConfiguration(conf, null, null); + ApplicationSubmissionContext appContext = Records + .newRecord(ApplicationSubmissionContext.class); + Collection<String> tagsFromConf = + amconfig.getTezConfiguration().getTrimmedStringCollection( + TezConfiguration.TEZ_APPLICATION_TAGS); + appContext.setApplicationTags(new HashSet<String>(tagsFromConf)); + assertTrue(appContext.getApplicationTags().contains("foo")); + assertTrue(appContext.getApplicationTags().contains("bar")); + } + @Test(timeout = 5000) public void testSessionTokenInAmClc() throws IOException, YarnException { http://git-wip-us.apache.org/repos/asf/tez/blob/de7fd9aa/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java index 1335aa2..a826f6d 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java @@ -84,6 +84,8 @@ public class DeprecatedKeys { TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH); mrParamToDAGParamMap.put(MRJobConfig.TASK_TIMEOUT, TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS); + mrParamToDAGParamMap.put(MRJobConfig.JOB_TAGS, + TezConfiguration.TEZ_APPLICATION_TAGS); } // TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current initialization in MRAppMaster, TezChild. http://git-wip-us.apache.org/repos/asf/tez/blob/de7fd9aa/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java index d44be30..7db98bc 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java @@ -64,6 +64,8 @@ public interface MRJobConfig { public static final String QUEUE_NAME = "mapreduce.job.queuename"; + public static final String JOB_TAGS = "mapreduce.job.tags"; + public static final String JVM_NUMTASKS_TORUN = "mapreduce.job.jvm.numtasks"; public static final String SPLIT_FILE = "mapreduce.job.splitfile";
