Updated Branches: refs/heads/TEZ-1 eeaaf232f -> 865232927
Fix build after YARN-717, YARN-720. (sseth) Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/86523292 Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/86523292 Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/86523292 Branch: refs/heads/TEZ-1 Commit: 8652329273978266490806028a4c274af33d213e Parents: eeaaf23 Author: Siddharth Seth <[email protected]> Authored: Mon Jun 3 09:25:46 2013 -0700 Committer: Siddharth Seth <[email protected]> Committed: Mon Jun 3 09:25:46 2013 -0700 ---------------------------------------------------------------------- .../dag/app/launcher/ContainerLauncherImpl.java | 13 ++++++------- .../apache/tez/dag/app/rm/NMCommunicatorEvent.java | 8 ++++---- .../dag/app/rm/NMCommunicatorStopRequestEvent.java | 4 ++-- .../java/org/apache/tez/mapreduce/YARNRunner.java | 7 ++----- 4 files changed, 14 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java index 5b23c75..b71af02 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java @@ -36,7 +36,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.mapred.ShuffleHandler; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.Clock; import org.apache.hadoop.yarn.YarnException; @@ -46,7 +45,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.service.AbstractService; @@ -123,10 +122,10 @@ public class ContainerLauncherImpl extends AbstractService implements // store enough information to be able to cleanup the container private ContainerId containerID; final private String containerMgrAddress; - private ContainerToken containerToken; + private Token containerToken; public Container(ContainerId containerID, - String containerMgrAddress, ContainerToken containerToken) { + String containerMgrAddress, Token containerToken) { this.state = ContainerState.PREP; this.containerMgrAddress = containerMgrAddress; this.containerID = containerID; @@ -355,7 +354,7 @@ public class ContainerLauncherImpl extends AbstractService implements } protected ContainerManager getCMProxy(ContainerId containerID, - final String containerManagerBindAddr, ContainerToken containerToken) + final String containerManagerBindAddr, Token containerToken) throws IOException { final InetSocketAddress cmAddr = @@ -364,8 +363,8 @@ public class ContainerLauncherImpl extends AbstractService implements // the user in createRemoteUser in this context has to be ContainerID UserGroupInformation user = UserGroupInformation .createRemoteUser(containerID.toString()); - Token<ContainerTokenIdentifier> token = ProtoUtils.convertFromProtoFormat( - containerToken, cmAddr); + org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token = ProtoUtils + .convertFromProtoFormat(containerToken, cmAddr); user.addToken(token); ContainerManager proxy = user http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java index 38f54c6..8bdeb28 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java @@ -19,18 +19,18 @@ package org.apache.tez.dag.app.rm; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.event.AbstractEvent; public class NMCommunicatorEvent extends AbstractEvent<NMCommunicatorEventType> { private final ContainerId containerId; private final NodeId nodeId; - private final ContainerToken containerToken; + private final Token containerToken; public NMCommunicatorEvent(ContainerId containerId, NodeId nodeId, - ContainerToken containerToken, NMCommunicatorEventType type) { + Token containerToken, NMCommunicatorEventType type) { super(type); this.containerId = containerId; this.nodeId = nodeId; @@ -45,7 +45,7 @@ public class NMCommunicatorEvent extends AbstractEvent<NMCommunicatorEventType> return this.nodeId; } - public ContainerToken getContainerToken() { + public Token getContainerToken() { return this.containerToken; } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java index 8f09a48..277d1e7 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java @@ -19,13 +19,13 @@ package org.apache.tez.dag.app.rm; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Token; public class NMCommunicatorStopRequestEvent extends NMCommunicatorEvent { public NMCommunicatorStopRequestEvent(ContainerId containerId, NodeId nodeId, - ContainerToken containerToken) { + Token containerToken) { super(containerId, nodeId, containerToken, NMCommunicatorEventType.CONTAINER_STOP_REQUEST); } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java ---------------------------------------------------------------------- diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java index 801ce8a..572437b 100644 --- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java +++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java @@ -70,7 +70,6 @@ import org.apache.hadoop.mapreduce.v2.LogParams; import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.YarnException; @@ -1141,13 +1140,11 @@ public class YARNRunner implements ClientProtocol { private static void addLog4jSystemProperties(String logLevel, List<String> vargs) { vargs.add("-Dlog4j.configuration=container-log4j.properties"); - // TODO Fix the remaining properties after YARN-720 is fixed. - // May need to introduce a log4j properties file for tez tasks. - vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "=" + vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); // Setting this to 0 to avoid log size restrictions. // Should be enforced by YARN. - vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + 0); + vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=" + 0); vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA"); }
