Repository: tez
Updated Branches:
refs/heads/branch-0.7 07e19ec96 -> 1d3e4a8f4
TEZ-909. Provide support for application tags (ebadger via rohini)
Conflicts:
CHANGES.txt
tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1d3e4a8f
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1d3e4a8f
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1d3e4a8f
Branch: refs/heads/branch-0.7
Commit: 1d3e4a8f4e2047ad8eb88b0f08402fe668f9da1a
Parents: 07e19ec
Author: Rohini Palaniswamy <[email protected]>
Authored: Fri May 27 12:32:21 2016 -0700
Committer: Rohini Palaniswamy <[email protected]>
Committed: Fri May 27 12:52:24 2016 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/client/TezClientUtils.java | 7 +++++++
.../org/apache/tez/dag/api/TezConfiguration.java | 7 +++++++
.../org/apache/tez/client/TestTezClientUtils.java | 18 ++++++++++++++++++
.../tez/mapreduce/hadoop/DeprecatedKeys.java | 2 ++
.../apache/tez/mapreduce/hadoop/MRJobConfig.java | 2 ++
6 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/1d3e4a8f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 6221783..5dde7f1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ Release 0.7.2 Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-909. Provide support for application tags
TEZ-2769. TEZ-UI Hive SQL is only displayed to line 11
TEZ-3257. Fix flaky test TestUnorderedPartitionedKVWriter.
TEZ-3240. Improvements to tez.lib.uris to allow for multiple tarballs and
mixing tarballs and jars.
http://git-wip-us.apache.org/repos/asf/tez/blob/1d3e4a8f/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 8a5a661..7a5bf7c 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -700,7 +700,14 @@ public class TezClientUtils {
ApplicationSubmissionContext appContext = Records
.newRecord(ApplicationSubmissionContext.class);
+ Collection<String> tagsFromConf =
+ amConfig.getTezConfiguration().getTrimmedStringCollection(
+ TezConfiguration.TEZ_APPLICATION_TAGS);
+
appContext.setApplicationType(TezConstants.TEZ_APPLICATION_TYPE);
+ if (tagsFromConf != null && !tagsFromConf.isEmpty()) {
+ appContext.setApplicationTags(new HashSet<String>(tagsFromConf));
+ }
appContext.setApplicationId(appId);
appContext.setResource(capability);
if (amConfig.getQueueName() != null) {
http://git-wip-us.apache.org/repos/asf/tez/blob/1d3e4a8f/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 6bb531b..45aac5d 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1054,6 +1054,13 @@ public class TezConfiguration extends Configuration {
@ConfigurationScope(Scope.AM)
public static final String TEZ_QUEUE_NAME = TEZ_PREFIX + "queue.name";
+ /**
+ * String value. Tags for the job that will be passed to YARN at submission
+ * time. Queries to YARN for applications can filter on these tags.
+ */
+ @ConfigurationScope(Scope.AM)
+ public static final String TEZ_APPLICATION_TAGS = TEZ_PREFIX +
"application.tags";
+
@Unstable
/**
* Boolean value. Generate debug artifacts like DAG plan in text format.
http://git-wip-us.apache.org/repos/asf/tez/blob/1d3e4a8f/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git
a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index 70a693a..c630db4 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -32,7 +32,9 @@ import java.net.URLClassLoader;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -58,6 +60,7 @@ import
org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Records;
import org.apache.tez.common.security.HistoryACLPolicyManager;
import org.apache.tez.common.security.JobTokenIdentifier;
import org.apache.tez.common.security.JobTokenSecretManager;
@@ -291,6 +294,21 @@ public class TestTezClientUtils {
Assert.assertTrue(localFs.delete(topDir, true));
}
+ @Test(timeout=1000)
+ public void testSetApplicationTags() {
+ TezConfiguration conf = new TezConfiguration(false);
+ conf.set(TezConfiguration.TEZ_APPLICATION_TAGS, "foo,bar");
+ AMConfiguration amconfig = new AMConfiguration(conf, null, null);
+ ApplicationSubmissionContext appContext = Records
+ .newRecord(ApplicationSubmissionContext.class);
+ Collection<String> tagsFromConf =
+ amconfig.getTezConfiguration().getTrimmedStringCollection(
+ TezConfiguration.TEZ_APPLICATION_TAGS);
+ appContext.setApplicationTags(new HashSet<String>(tagsFromConf));
+ assertTrue(appContext.getApplicationTags().contains("foo"));
+ assertTrue(appContext.getApplicationTags().contains("bar"));
+ }
+
@Test(timeout = 5000)
public void testSessionTokenInAmClc() throws IOException, YarnException {
http://git-wip-us.apache.org/repos/asf/tez/blob/1d3e4a8f/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git
a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index 1335aa2..a826f6d 100644
---
a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -84,6 +84,8 @@ public class DeprecatedKeys {
TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);
mrParamToDAGParamMap.put(MRJobConfig.TASK_TIMEOUT,
TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS);
+ mrParamToDAGParamMap.put(MRJobConfig.JOB_TAGS,
+ TezConfiguration.TEZ_APPLICATION_TAGS);
}
// TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of
the current initialization in MRAppMaster, TezChild.
http://git-wip-us.apache.org/repos/asf/tez/blob/1d3e4a8f/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
----------------------------------------------------------------------
diff --git
a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
index d44be30..7db98bc 100644
---
a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
+++
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
@@ -64,6 +64,8 @@ public interface MRJobConfig {
public static final String QUEUE_NAME = "mapreduce.job.queuename";
+ public static final String JOB_TAGS = "mapreduce.job.tags";
+
public static final String JVM_NUMTASKS_TORUN = "mapreduce.job.jvm.numtasks";
public static final String SPLIT_FILE = "mapreduce.job.splitfile";
