Repository: tez Updated Branches: refs/heads/branch-0.6 da85ca8cf -> a03e1c951
TEZ-2385. branch-0.6 compile failure caused by TEZ-2226. (hitesh) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a03e1c95 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a03e1c95 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a03e1c95 Branch: refs/heads/branch-0.6 Commit: a03e1c951782372857a2431c41bef570b3ac8e10 Parents: da85ca8 Author: Hitesh Shah <[email protected]> Authored: Wed Apr 29 14:38:40 2015 -0700 Committer: Hitesh Shah <[email protected]> Committed: Wed Apr 29 14:38:40 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../main/java/org/apache/tez/dag/api/DAG.java | 34 ++++++++++++++------ .../apache/tez/dag/api/TezConfiguration.java | 2 -- 3 files changed, 25 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/a03e1c95/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 5ba43b6..8b8d4f5 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,7 @@ Release 0.6.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2385. branch-0.6 compile failure caused by TEZ-2226. TEZ-2226. Disable writing history to timeline if domain creation fails. TEZ-2259. Push additional data to Timeline for Recovery for better consumption in UI. TEZ-2365. Update tez-ui war's license/notice to reflect OFL license correctly. http://git-wip-us.apache.org/repos/asf/tez/blob/a03e1c95/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java index 2779286..906b1b7 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java @@ -84,6 +84,7 @@ public class DAG { Credentials credentials = new Credentials(); Set<VertexGroup> vertexGroups = Sets.newHashSet(); Set<GroupInputEdge> groupInputEdges = Sets.newHashSet(); + private Map<String, String> dagConf = new HashMap<String, String>(); private DAGAccessControls dagAccessControls; Map<String, LocalResource> commonTaskLocalFiles = Maps.newHashMap(); @@ -319,7 +320,12 @@ public class DAG { public String getName() { return this.name; } - + + public DAG setConf(String property, String value) { + dagConf.put(property, value); + return this; + } + @Private @VisibleForTesting public Map<String,String> getDagConf() { @@ -670,16 +676,16 @@ public class DAG { } // create protobuf message describing DAG - public DAGPlan createDag(Configuration dagConf, Credentials extraCredentials, + public DAGPlan createDag(Configuration conf, Credentials extraCredentials, Map<String, LocalResource> tezJarResources, LocalResource binaryConfig, boolean tezLrsAsArchive) { - return createDag(dagConf, extraCredentials, tezJarResources, binaryConfig, tezLrsAsArchive, + return createDag(conf, extraCredentials, tezJarResources, binaryConfig, tezLrsAsArchive, null); } // create protobuf message describing DAG @Private - public DAGPlan createDag(Configuration dagConf, Credentials extraCredentials, + public DAGPlan createDag(Configuration tezConf, Credentials extraCredentials, Map<String, LocalResource> tezJarResources, LocalResource binaryConfig, boolean tezLrsAsArchive, Map<String, String> additionalConfigs) { verify(true); @@ -728,9 +734,9 @@ public class DAG { // infer credentials, resources and parallelism from data source Resource vertexTaskResource = vertex.getTaskResource(); if (vertexTaskResource == null) { - vertexTaskResource = Resource.newInstance(dagConf.getInt( + vertexTaskResource = Resource.newInstance(tezConf.getInt( TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, - TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT), dagConf.getInt( + TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT), tezConf.getInt( TezConfiguration.TEZ_TASK_RESOURCE_CPU_VCORES, TezConfiguration.TEZ_TASK_RESOURCE_CPU_VCORES_DEFAULT)); } @@ -801,7 +807,7 @@ public class DAG { taskConfigBuilder.setMemoryMb(vertexTaskResource.getMemory()); taskConfigBuilder.setVirtualCores(vertexTaskResource.getVirtualCores()); taskConfigBuilder.setJavaOpts( - TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vertex.getTaskLaunchCmdOpts(), dagConf)); + TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vertex.getTaskLaunchCmdOpts(), tezConf)); taskConfigBuilder.setTaskModule(vertex.getName()); if (!vertexLRs.isEmpty()) { @@ -809,7 +815,7 @@ public class DAG { } Map<String, String> taskEnv = Maps.newHashMap(vertex.getTaskEnvironment()); - TezYARNUtils.setupDefaultEnv(taskEnv, dagConf, + TezYARNUtils.setupDefaultEnv(taskEnv, tezConf, TezConfiguration.TEZ_TASK_LAUNCH_ENV, TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT, tezLrsAsArchive); for (Map.Entry<String, String> entry : taskEnv.entrySet()) { @@ -879,8 +885,8 @@ public class DAG { ConfigurationProto.Builder confProtoBuilder = ConfigurationProto.newBuilder(); - if (dagConf != null) { - Iterator<Entry<String, String>> iter = dagConf.iterator(); + if (tezConf != null) { + Iterator<Entry<String, String>> iter = tezConf.iterator(); while (iter.hasNext()) { Entry<String, String> entry = iter.next(); PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder(); @@ -909,6 +915,14 @@ public class DAG { confProtoBuilder.addConfKeyValues(kvp); } } + if (this.dagConf != null && !this.dagConf.isEmpty()) { + for (Entry<String, String> entry : this.dagConf.entrySet()) { + PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder(); + kvp.setKey(entry.getKey()); + kvp.setValue(entry.getValue()); + confProtoBuilder.addConfKeyValues(kvp); + } + } dagBuilder.setDagKeyValues(confProtoBuilder); // This does not seem to be used anywhere // should this replace BINARY_PB_CONF??? http://git-wip-us.apache.org/repos/asf/tez/blob/a03e1c95/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index c5a3615..debc269 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -170,7 +170,6 @@ public class TezConfiguration extends Configuration { /** * Boolean value. Determine whether to log history events per dag */ - @ConfigurationScope(Scope.DAG) @Private public static final String TEZ_DAG_HISTORY_LOGGING_ENABLED = TEZ_PREFIX + "dag.history.logging.enabled"; @@ -631,7 +630,6 @@ public class TezConfiguration extends Configuration { * Boolean value. Allow/disable logging for all dags in a session */ @Private - @ConfigurationScope(Scope.AM) public static final String TEZ_AM_HISTORY_LOGGING_ENABLED = TEZ_AM_PREFIX + "history.logging.enabled"; public static final boolean TEZ_AM_HISTORY_LOGGING_ENABLED_DEFAULT = true;
