TEZ-178. Change YARN object construction to be based on newInstance in various classes instead of BuilderUtils. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/9f040cf8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/9f040cf8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/9f040cf8 Branch: refs/heads/master Commit: 9f040cf8b33600233be2bd2b9795406816c687e4 Parents: 8652329 Author: Siddharth Seth <[email protected]> Authored: Mon Jun 3 13:35:56 2013 -0700 Committer: Siddharth Seth <[email protected]> Committed: Mon Jun 3 13:35:56 2013 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/tez/common/IDUtils.java | 5 +- .../java/org/apache/tez/dag/records/TezDAGID.java | 5 +- .../org/apache/tez/records/TezContainerId.java | 11 ++- .../org/apache/tez/dag/api/DagTypeConverters.java | 5 +- .../org/apache/tez/dag/api/oldrecords/AMInfo.java | 2 +- .../org/apache/tez/dag/app/MRRExampleHelper.java | 10 +- .../apache/tez/dag/app/TaskHeartbeatHandler.java | 2 +- .../tez/dag/app/dag/TaskAttemptStateInternal.java | 2 +- .../TaskAttemptEventContainerTerminating.java | 2 +- .../app/dag/event/TaskAttemptEventFailRequest.java | 2 +- .../app/dag/event/TaskAttemptEventNodeFailed.java | 2 +- .../tez/dag/app/dag/impl/DAGSchedulerMRR.java | 8 +- .../dag/app/dag/impl/DAGSchedulerNaturalOrder.java | 8 +- .../tez/dag/app/rm/AMSchedulerEventTAEnded.java | 2 +- .../app/rm/AMSchedulerEventTALaunchRequest.java | 2 +- .../rm/container/AMContainerEventNodeFailed.java | 2 +- .../dag/app/rm/container/AMContainerHelpers.java | 3 +- .../tez/dag/app/rm/container/AMContainerMap.java | 2 +- .../app/rm/node/AMNodeEventTaskAttemptEnded.java | 2 +- .../tez/dag/app/rm/node/AMNodeEventType.java | 2 +- .../org/apache/tez/dag/app/rm/node/AMNodeMap.java | 2 +- .../org/apache/tez/dag/utils/TezBuilderUtils.java | 2 +- .../apache/tez/dag/app/dag/impl/TestDAGImpl.java | 6 +- .../tez/dag/app/dag/impl/TestTaskAttempt.java | 74 +++++++------- .../apache/tez/dag/app/dag/impl/TestTaskImpl.java | 5 +- .../tez/dag/app/dag/impl/TestVertexImpl.java | 6 +- .../apache/tez/dag/app/rm/TestTaskScheduler.java | 6 +- .../tez/dag/app/rm/container/TestAMContainer.java | 15 ++-- .../apache/tez/mapreduce/hadoop/IDConverter.java | 3 +- .../hadoop/MultiStageMRConfToTezTranslator.java | 2 +- .../org/apache/tez/mapreduce/TezTestUtils.java | 10 +- .../org/apache/tez/mapreduce/NotRunningJob.java | 1 - .../java/org/apache/tez/mapreduce/YARNRunner.java | 7 +- 33 files changed, 107 insertions(+), 111 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-common/src/main/java/org/apache/tez/common/IDUtils.java ---------------------------------------------------------------------- diff --git a/tez-common/src/main/java/org/apache/tez/common/IDUtils.java b/tez-common/src/main/java/org/apache/tez/common/IDUtils.java index e2cbf35..e94d939 100644 --- a/tez-common/src/main/java/org/apache/tez/common/IDUtils.java +++ b/tez-common/src/main/java/org/apache/tez/common/IDUtils.java @@ -19,7 +19,6 @@ package org.apache.tez.common; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.tez.dag.records.TezDAGID; import org.apache.tez.dag.records.TezID; import org.apache.tez.dag.records.TezTaskAttemptID; @@ -41,7 +40,7 @@ public class IDUtils { String[] parts = str.split("_"); if(parts.length == 6) { if(parts[0].equals(TezTaskID.TASK)) { - ApplicationId appId = BuilderUtils.newApplicationId( + ApplicationId appId = ApplicationId.newInstance( Long.valueOf(parts[1]), Integer.parseInt(parts[2])); TezDAGID dagId = new TezDAGID(appId, Integer.parseInt(parts[3])); TezVertexID vId = new TezVertexID(dagId, Integer.parseInt(parts[4])); @@ -71,7 +70,7 @@ public class IDUtils { String[] parts = str.split(Character.toString(TezID.SEPARATOR)); if(parts.length == 7) { if(parts[0].equals(TezTaskAttemptID.ATTEMPT)) { - ApplicationId appId = BuilderUtils.newApplicationId( + ApplicationId appId = ApplicationId.newInstance( Long.valueOf(parts[1]), Integer.parseInt(parts[2])); TezDAGID dagId = new TezDAGID(appId, Integer.parseInt(parts[3])); TezVertexID vId = new TezVertexID(dagId, Integer.parseInt(parts[4])); http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java ---------------------------------------------------------------------- diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java index 733a908..9a0a409 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.text.NumberFormat; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.util.BuilderUtils; /** @@ -71,7 +70,7 @@ public class TezDAGID extends TezID { * @param id the dag number */ public TezDAGID(String yarnRMIdentifier, int appId, int id) { - this(BuilderUtils.newApplicationId(Long.valueOf(yarnRMIdentifier), + this(ApplicationId.newInstance(Long.valueOf(yarnRMIdentifier), appId), id); } @@ -117,7 +116,7 @@ public class TezDAGID extends TezID { @Override public void readFields(DataInput in) throws IOException { - applicationId = BuilderUtils.newApplicationId(in.readLong(), in.readInt()); + applicationId = ApplicationId.newInstance(in.readLong(), in.readInt()); super.readFields(in); } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java ---------------------------------------------------------------------- diff --git a/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java b/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java index 24c1e8a..193b979 100644 --- a/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java +++ b/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java @@ -24,8 +24,9 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.util.BuilderUtils; //TODO EVENTUALLY Once everything is on PB, get rid of this. //Alternately have the PB interfaces implement Writable. @@ -58,8 +59,12 @@ public class TezContainerId implements Writable { int appId = in.readInt(); int appAttemptId = in.readInt(); int id = in.readInt(); - this.containerId = BuilderUtils.newContainerId(appId, appAttemptId, - timestamp, id); + + ApplicationId applicationId = ApplicationId.newInstance(timestamp, appId); + ApplicationAttemptId applicationAttemptId = ApplicationAttemptId + .newInstance(applicationId, appAttemptId); + + this.containerId = ContainerId.newInstance(applicationAttemptId, id); } @Override http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java ---------------------------------------------------------------------- diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java index ba42e23..6437080 100644 --- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java +++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java @@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.tez.dag.api.records.DAGProtos.EdgePlan; import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeConnectionPattern; @@ -207,7 +206,7 @@ public class DagTypeConverters { public static Resource CreateResourceRequestFromTaskConfig( PlanTaskConfiguration taskConfig) { - return BuilderUtils.newResource(taskConfig.getMemoryMb(), taskConfig.getVirtualCores()); + return Resource.newInstance(taskConfig.getMemoryMb(), taskConfig.getVirtualCores()); } public static Map<String, String> createSettingsMapFromDAGPlan( @@ -218,4 +217,4 @@ public class DagTypeConverters { } return map; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java index 7726d7b..6cb3df0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java @@ -35,4 +35,4 @@ public interface AMInfo { public void setNodeManagerHost(String nmHost); public void setNodeManagerPort(int nmPort); public void setNodeManagerHttpPort(int mnHttpPort); -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java index 92f0e87..35ae558 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java @@ -16,7 +16,6 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.tez.dag.api.records.DAGProtos.DAGPlan; import org.apache.tez.dag.api.Edge; import org.apache.tez.dag.api.EdgeProperty; @@ -126,12 +125,13 @@ public class MRRExampleHelper { reduce2Rsrcs.put(MRJobConfig.JOB_CONF_FILE, jobRsrcs.get(MRJobConfig.JOB_CONF_FILE)); reduce2Rsrcs.put(getConfFileName("reduce2"), jobRsrcs.get(getConfFileName("reduce2"))); - Resource mapResource = BuilderUtils.newResource( + Resource mapResource = Resource.newInstance( MRJobConfig.DEFAULT_MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_CPU_VCORES); + mapVertex.setTaskResource(mapResource); mapVertex.setTaskLocalResources(mapRsrcs); - Resource reduceResource = BuilderUtils.newResource( + Resource reduceResource = Resource.newInstance( MRJobConfig.DEFAULT_REDUCE_MEMORY_MB, MRJobConfig.DEFAULT_REDUCE_CPU_VCORES); reduce1Vertex.setTaskResource(reduceResource); @@ -182,12 +182,12 @@ public class MRRExampleHelper { reduceRsrcs.put(MRJobConfig.JOB_CONF_FILE, jobRsrcs.get(MRJobConfig.JOB_CONF_FILE)); reduceRsrcs.put(getConfFileName("reduce"), jobRsrcs.get(MRJobConfig.JOB_CONF_FILE)); - Resource mapResource = BuilderUtils.newResource( + Resource mapResource = Resource.newInstance( MRJobConfig.DEFAULT_MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_CPU_VCORES); mapVertex.setTaskResource(mapResource); mapVertex.setTaskLocalResources(mapRsrcs); - Resource reduceResource = BuilderUtils.newResource( + Resource reduceResource = Resource.newInstance( MRJobConfig.DEFAULT_REDUCE_MEMORY_MB, MRJobConfig.DEFAULT_REDUCE_CPU_VCORES); reduceVertex.setTaskResource(reduceResource); http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java index 7239484..b6fdc53 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java @@ -65,4 +65,4 @@ public class TaskHeartbeatHandler extends HeartbeatHandlerBase<TezTaskAttemptID> eventHandler.handle(new TaskAttemptEvent(attemptId, TaskAttemptEventType.TA_TIMED_OUT)); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java index 3633b08..9ad5460 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java @@ -36,4 +36,4 @@ public enum TaskAttemptStateInternal { KILLED, FAILED, SUCCEEDED -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java index 8c88cb9..7da6e14 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java @@ -35,4 +35,4 @@ public class TaskAttemptEventContainerTerminating extends TaskAttemptEvent return this.message; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java index bba8027..b2f9cfb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java @@ -32,4 +32,4 @@ public class TaskAttemptEventFailRequest extends TaskAttemptEvent { public String getMessage() { return this.message; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java index 4fc718d..ee143bb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java @@ -33,4 +33,4 @@ public class TaskAttemptEventNodeFailed extends TaskAttemptEvent { return this.message; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java index d404155..4f51c02 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java @@ -20,8 +20,8 @@ package org.apache.tez.dag.app.dag.impl; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.app.dag.DAG; import org.apache.tez.dag.app.dag.DAGScheduler; @@ -124,9 +124,9 @@ public class DAGSchedulerMRR implements DAGScheduler { " with depth " + vertexDistanceFromRoot + " at priority " + priority); - TaskAttemptEventSchedule attemptEvent = - new TaskAttemptEventSchedule(attempt.getID(), - BuilderUtils.newPriority(priority)); + TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule( + attempt.getID(), Priority.newInstance(priority)); + sendEvent(attemptEvent); } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java index 9e8729e..6fec907 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java @@ -20,8 +20,8 @@ package org.apache.tez.dag.app.dag.impl; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.tez.dag.app.dag.DAG; import org.apache.tez.dag.app.dag.DAGScheduler; import org.apache.tez.dag.app.dag.TaskAttempt; @@ -66,9 +66,9 @@ public class DAGSchedulerNaturalOrder implements DAGScheduler { LOG.info("Scheduling " + attempt.getID() + " at priority " + priority); - TaskAttemptEventSchedule attemptEvent = - new TaskAttemptEventSchedule(attempt.getID(), - BuilderUtils.newPriority(priority)); + TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule( + attempt.getID(), Priority.newInstance(priority)); + sendEvent(attemptEvent); } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java index b2a90f2..90e76b7 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java @@ -51,4 +51,4 @@ public class AMSchedulerEventTAEnded extends AMSchedulerEvent { public ContainerId getUsedContainerId() { return this.containerId; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java index b595c2b..313ee7d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java @@ -130,4 +130,4 @@ public class AMSchedulerEventTALaunchRequest extends AMSchedulerEvent { * - Credentials, tokens etc are identical. * + Command - dependent on map / reduce java.opts */ -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java index c1c603c..9f5d39c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java @@ -36,4 +36,4 @@ public class AMContainerEventNodeFailed extends AMContainerEvent implements return message; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java index c17f7f2..6156e33 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java @@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.app.AppContext; @@ -76,7 +75,7 @@ public class AMContainerHelpers { long resourceSize = fstat.getLen(); long resourceModificationTime = fstat.getModificationTime(); - return BuilderUtils.newLocalResource(resourceURL, type, visibility, + return LocalResource.newInstance(resourceURL, type, visibility, resourceSize, resourceModificationTime); } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java index 708838c..5f5a7be 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java @@ -72,4 +72,4 @@ public class AMContainerMap extends AbstractService implements public Collection<AMContainer> values() { return containerMap.values(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java index cc2b7f7..c823236 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java @@ -50,4 +50,4 @@ public class AMNodeEventTaskAttemptEnded extends AMNodeEvent { public TezTaskAttemptID getTaskAttemptId() { return this.taskAttemptId; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java index 60534fd..421e482 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java @@ -38,4 +38,4 @@ public enum AMNodeEventType { // Producer: AMNode - Will not reach AMNodeImpl. Used to compute whether // blacklisting should be ignored. N_NODE_WAS_BLACKLISTED -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java index 206244e..a9256aa 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java @@ -210,4 +210,4 @@ public class AMNodeMap extends AbstractService implements public boolean isBlacklistingIgnored() { return this.ignoreBlacklisting; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java index 60ec0a0..a734bb8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java @@ -51,4 +51,4 @@ public class TezBuilderUtils { return new TezTaskID(newVertexID(dagId, vertexId), taskId); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java index 9e94cbd..9588099 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java @@ -27,9 +27,9 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.Clock; import org.apache.hadoop.yarn.SystemClock; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.oldrecords.TaskState; import org.apache.tez.dag.api.records.DAGProtos.DAGPlan; @@ -343,8 +343,8 @@ public class TestDAGImpl { @Before public void setup() { conf = new TezConfiguration(); - appAttemptId = BuilderUtils.newApplicationAttemptId( - BuilderUtils.newApplicationId(100, 1), 1); + appAttemptId = ApplicationAttemptId.newInstance( + ApplicationId.newInstance(100, 1), 1); dagId = new TezDAGID(appAttemptId.getApplicationId(), 1); Assert.assertNotNull(dagId); dagPlan = createTestDAGPlan(); http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java index 9601418..516c656 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java @@ -49,10 +49,10 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.tez.common.TezTaskContext; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint; @@ -131,7 +131,7 @@ public class TestTaskAttempt { mock(TaskAttemptListener.class), 1, new TezConfiguration(), mock(Token.class), new Credentials(), new SystemClock(), mock(TaskHeartbeatHandler.class), mock(AppContext.class), - MAP_PROCESSOR_NAME, locationHint, BuilderUtils.newResource(1024, 1), + MAP_PROCESSOR_NAME, locationHint, Resource.newInstance(1024, 1), new HashMap<String, LocalResource>(), new HashMap<String, String>(), "", false); @@ -175,7 +175,7 @@ public class TestTaskAttempt { mock(TaskAttemptListener.class), 1, new TezConfiguration(), mock(Token.class), new Credentials(), new SystemClock(), mock(TaskHeartbeatHandler.class), mock(AppContext.class), - MAP_PROCESSOR_NAME, locationHint, BuilderUtils.newResource(1024, 1), + MAP_PROCESSOR_NAME, locationHint, Resource.newInstance(1024, 1), new HashMap<String, LocalResource>(), new HashMap<String, String>(), "", false); TaskAttemptImpl spyTa = spy(taImpl); @@ -222,8 +222,8 @@ public class TestTaskAttempt { // Configuration conf = new Configuration(); // conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb); // conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb); - // app.setClusterInfo(new ClusterInfo(BuilderUtils - // .newResource(minContainerSize, 1), BuilderUtils.newResource(10240,1))); + // app.setClusterInfo(new ClusterInfo(Resource.newInstance(minContainerSize, 1), + // Resource.newInstance(10240,1))); // // Job job = app.submit(conf); // app.waitForState(job, JobState.RUNNING); @@ -294,8 +294,8 @@ public class TestTaskAttempt { // Ensure the dag does not go into an error state if a attempt kill is // received while STARTING public void testLaunchFailedWhileKilling() throws Exception { - ApplicationId appId = BuilderUtils.newApplicationId(1, 2); - ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + ApplicationId appId = ApplicationId.newInstance(1, 2); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( appId, 0); TezDAGID dagID = new TezDAGID(appId, 1); TezVertexID vertexID = new TezVertexID(dagID, 1); @@ -313,7 +313,7 @@ public class TestTaskAttempt { TaskLocationHint locationHint = new TaskLocationHint( new String[] { "127.0.0.1" }, null); - Resource resource = BuilderUtils.newResource(1024, 1); + Resource resource = Resource.newInstance(1024, 1); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); Map<String, String> environment = new HashMap<String, String>(); String javaOpts = ""; @@ -327,14 +327,14 @@ public class TestTaskAttempt { MAP_PROCESSOR_NAME, locationHint, resource, localResources, environment, javaOpts, false); - NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0); - ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3); + NodeId nid = NodeId.newInstance("127.0.0.1", 0); + ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); when(container.getId()).thenReturn(contId); when(container.getNodeId()).thenReturn(nid); - taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, BuilderUtils - .newPriority(3))); + taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, Priority + .newInstance(3))); // At state STARTING. taImpl.handle(new TaskAttemptEventKillRequest(taskAttemptID, null)); // At some KILLING state. @@ -348,8 +348,8 @@ public class TestTaskAttempt { // Ensure ContainerTerminating and ContainerTerminated is handled correctly by // the TaskAttempt public void testContainerTerminationWhileRunning() throws Exception { - ApplicationId appId = BuilderUtils.newApplicationId(1, 2); - ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + ApplicationId appId = ApplicationId.newInstance(1, 2); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( appId, 0); TezDAGID dagID = new TezDAGID(appId, 1); TezVertexID vertexID = new TezVertexID(dagID, 1); @@ -367,13 +367,13 @@ public class TestTaskAttempt { TaskLocationHint locationHint = new TaskLocationHint( new String[] { "127.0.0.1" }, null); - Resource resource = BuilderUtils.newResource(1024, 1); + Resource resource = Resource.newInstance(1024, 1); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); Map<String, String> environment = new HashMap<String, String>(); String javaOpts = ""; - NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0); - ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3); + NodeId nid = NodeId.newInstance("127.0.0.1", 0); + ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); when(container.getId()).thenReturn(contId); when(container.getNodeId()).thenReturn(nid); @@ -443,8 +443,8 @@ public class TestTaskAttempt { @Test // Ensure ContainerTerminated is handled correctly by the TaskAttempt public void testContainerTerminatedWhileRunning() throws Exception { - ApplicationId appId = BuilderUtils.newApplicationId(1, 2); - ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + ApplicationId appId = ApplicationId.newInstance(1, 2); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( appId, 0); TezDAGID dagID = new TezDAGID(appId, 1); TezVertexID vertexID = new TezVertexID(dagID, 1); @@ -462,13 +462,13 @@ public class TestTaskAttempt { TaskLocationHint locationHint = new TaskLocationHint( new String[] { "127.0.0.1" }, null); - Resource resource = BuilderUtils.newResource(1024, 1); + Resource resource = Resource.newInstance(1024, 1); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); Map<String, String> environment = new HashMap<String, String>(); String javaOpts = ""; - NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0); - ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3); + NodeId nid = NodeId.newInstance("127.0.0.1", 0); + ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); when(container.getId()).thenReturn(contId); when(container.getNodeId()).thenReturn(nid); @@ -508,8 +508,8 @@ public class TestTaskAttempt { @Test // Ensure ContainerTerminated is handled correctly by the TaskAttempt public void testContainerTerminatedWhileCommitting() throws Exception { - ApplicationId appId = BuilderUtils.newApplicationId(1, 2); - ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + ApplicationId appId = ApplicationId.newInstance(1, 2); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( appId, 0); TezDAGID dagID = new TezDAGID(appId, 1); TezVertexID vertexID = new TezVertexID(dagID, 1); @@ -527,13 +527,13 @@ public class TestTaskAttempt { TaskLocationHint locationHint = new TaskLocationHint( new String[] { "127.0.0.1" }, null); - Resource resource = BuilderUtils.newResource(1024, 1); + Resource resource = Resource.newInstance(1024, 1); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); Map<String, String> environment = new HashMap<String, String>(); String javaOpts = ""; - NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0); - ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3); + NodeId nid = NodeId.newInstance("127.0.0.1", 0); + ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); when(container.getId()).thenReturn(contId); when(container.getNodeId()).thenReturn(nid); @@ -575,8 +575,8 @@ public class TestTaskAttempt { // Ensure ContainerTerminating and ContainerTerminated is handled correctly by // the TaskAttempt public void testContainerTerminatedAfterSuccess() throws Exception { - ApplicationId appId = BuilderUtils.newApplicationId(1, 2); - ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + ApplicationId appId = ApplicationId.newInstance(1, 2); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( appId, 0); TezDAGID dagID = new TezDAGID(appId, 1); TezVertexID vertexID = new TezVertexID(dagID, 1); @@ -594,13 +594,13 @@ public class TestTaskAttempt { TaskLocationHint locationHint = new TaskLocationHint( new String[] { "127.0.0.1" }, null); - Resource resource = BuilderUtils.newResource(1024, 1); + Resource resource = Resource.newInstance(1024, 1); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); Map<String, String> environment = new HashMap<String, String>(); String javaOpts = ""; - NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0); - ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3); + NodeId nid = NodeId.newInstance("127.0.0.1", 0); + ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); when(container.getId()).thenReturn(contId); when(container.getNodeId()).thenReturn(nid); @@ -667,8 +667,8 @@ public class TestTaskAttempt { // Verifies that multiple TooManyFetchFailures are handled correctly by the // TaskAttempt. public void testMultipleTooManyFetchFailures() throws Exception { - ApplicationId appId = BuilderUtils.newApplicationId(1, 2); - ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + ApplicationId appId = ApplicationId.newInstance(1, 2); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( appId, 0); TezDAGID dagID = new TezDAGID(appId, 1); TezVertexID vertexID = new TezVertexID(dagID, 1); @@ -687,13 +687,13 @@ public class TestTaskAttempt { TaskLocationHint locationHint = new TaskLocationHint( new String[] { "127.0.0.1" }, null); - Resource resource = BuilderUtils.newResource(1024, 1); + Resource resource = Resource.newInstance(1024, 1); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); Map<String, String> environment = new HashMap<String, String>(); String javaOpts = ""; - NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0); - ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3); + NodeId nid = NodeId.newInstance("127.0.0.1", 0); + ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); when(container.getId()).thenReturn(contId); when(container.getNodeId()).thenReturn(nid); http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java index 44e7686..f2ab880 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java @@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.InlineDispatcher; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint; import org.apache.tez.dag.api.oldrecords.TaskAttemptState; @@ -101,11 +100,11 @@ public class TestTaskImpl { clock = new SystemClock(); locationHint = new TaskLocationHint(new String[1], new String[1]); - appId = BuilderUtils.newApplicationId(System.currentTimeMillis(), 1); + appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); dagId = new TezDAGID(appId, 1); vertexId = new TezVertexID(dagId, 1); appContext = mock(AppContext.class); - taskResource = BuilderUtils.newResource(1024, 1); + taskResource = Resource.newInstance(1024, 1); localResources = new HashMap<String, LocalResource>(); environment = new HashMap<String, String>(); javaOpts = ""; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java index 41b500f..e3af5b3 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java @@ -34,9 +34,9 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.Clock; import org.apache.hadoop.yarn.SystemClock; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.tez.dag.api.oldrecords.TaskState; import org.apache.tez.dag.api.records.DAGProtos.DAGPlan; import org.apache.tez.dag.api.records.DAGProtos.EdgePlan; @@ -463,8 +463,8 @@ public class TestVertexImpl { @Before public void setup() { - appAttemptId = BuilderUtils.newApplicationAttemptId( - BuilderUtils.newApplicationId(100, 1), 1); + appAttemptId = ApplicationAttemptId.newInstance( + ApplicationId.newInstance(100, 1), 1); dagId = new TezDAGID(appAttemptId.getApplicationId(), 1); dagPlan = createTestDAGPlan(); dispatcher = new DrainDispatcher(); http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java index 8d52e9a..7036e09 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -40,7 +41,6 @@ import org.apache.hadoop.yarn.client.AMRMClientAsync; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.RackResolver; import org.apache.tez.dag.app.rm.TaskScheduler.CookieContainerRequest; import org.apache.tez.dag.app.rm.TaskScheduler.TaskSchedulerAppCallback; @@ -68,8 +68,8 @@ public class TestTaskScheduler { mock(AMRMClientAsync.class); ApplicationAttemptId attemptId = - BuilderUtils.newApplicationAttemptId( - BuilderUtils.newApplicationId(1234, 0), 0); + ApplicationAttemptId.newInstance( + ApplicationId.newInstance(1234, 0), 0); String appHost = "host"; int appPort = 0; String appUrl = "url"; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java index cf01f43..332f737 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java @@ -51,7 +51,6 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.tez.common.TezTaskContext; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.app.AppContext; @@ -831,14 +830,14 @@ public class TestAMContainer { @SuppressWarnings("unchecked") public WrappedContainer() { - applicationID = BuilderUtils.newApplicationId(rmIdentifier, 1); - appAttemptID = BuilderUtils.newApplicationAttemptId(applicationID, 1); - containerID = BuilderUtils.newContainerId(appAttemptID, 1); - nodeID = BuilderUtils.newNodeId("host", 12500); + applicationID = ApplicationId.newInstance(rmIdentifier, 1); + appAttemptID = ApplicationAttemptId.newInstance(applicationID, 1); + containerID = ContainerId.newInstance(appAttemptID, 1); + nodeID = NodeId.newInstance("host", 12500); nodeHttpAddress = "host:12501"; - resource = BuilderUtils.newResource(1024, 1); - priority = BuilderUtils.newPriority(1); - container = BuilderUtils.newContainer(containerID, nodeID, + resource = Resource.newInstance(1024, 1); + priority = Priority.newInstance(1); + container = Container.newInstance(containerID, nodeID, nodeHttpAddress, resource, priority, null); chh = mock(ContainerHeartbeatHandler.class); http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java index 4046596..f7c4f50 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java @@ -23,7 +23,6 @@ import org.apache.hadoop.mapred.TaskAttemptID; import org.apache.hadoop.mapred.TaskID; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.tez.dag.records.TezDAGID; import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.dag.records.TezTaskID; @@ -55,7 +54,7 @@ public class IDConverter { // FIXME hardcoded assumption that one app is one dag public static TezDAGID fromMRJobId( org.apache.hadoop.mapreduce.JobID jobId) { - return new TezDAGID(BuilderUtils.newApplicationId( + return new TezDAGID(ApplicationId.newInstance( Long.valueOf(jobId.getJtIdentifier()), jobId.getId()), 1); } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java index c444314..9f1ae88 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java @@ -189,4 +189,4 @@ public class MultiStageMRConfToTezTranslator { } } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java index 2e823e6..9a3a666 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java @@ -17,7 +17,7 @@ */ package org.apache.tez.mapreduce; -import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.tez.dag.records.TezDAGID; import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.dag.records.TezTaskID; @@ -31,7 +31,7 @@ public class TezTestUtils { new TezTaskID( new TezVertexID( new TezDAGID( - BuilderUtils.newApplicationId(0, jobId), jobId), + ApplicationId.newInstance(0, jobId), jobId), vertexId), taskId) , taskAttemptId); @@ -41,20 +41,20 @@ public class TezTestUtils { int vertexId, int taskId) { return new TezTaskID( new TezVertexID(new TezDAGID( - BuilderUtils.newApplicationId(0, jobId), + ApplicationId.newInstance(0, jobId), jobId), vertexId), taskId); } public static TezDAGID getMockJobId(int jobId) { return new TezDAGID( - BuilderUtils.newApplicationId(0, jobId), jobId); + ApplicationId.newInstance(0, jobId), jobId); } public static TezVertexID getMockVertexId(int jobId, int vId) { return new TezVertexID( new TezDAGID( - BuilderUtils.newApplicationId(0, jobId), jobId), + ApplicationId.newInstance(0, jobId), jobId), vId); } } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java ---------------------------------------------------------------------- diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java index ba7ab5a..c29102b 100644 --- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java +++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java @@ -68,7 +68,6 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.util.BuilderUtils; public class NotRunningJob implements MRClientProtocol { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java ---------------------------------------------------------------------- diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java index 572437b..e2ef5b6 100644 --- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java +++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java @@ -89,7 +89,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.util.Apps; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.tez.dag.api.DAG; @@ -486,7 +485,7 @@ public class YARNRunner implements ClientProtocol { Map<String, String> reduceEnv = new HashMap<String, String>(); setupMapReduceEnv(conf, reduceEnv, false); - Resource reduceResource = BuilderUtils.newResource(conf.getInt( + Resource reduceResource = Resource.newInstance(conf.getInt( MRJobConfig.REDUCE_MEMORY_MB, MRJobConfig.DEFAULT_REDUCE_MEMORY_MB), conf.getInt(MRJobConfig.REDUCE_CPU_VCORES, MRJobConfig.DEFAULT_REDUCE_CPU_VCORES)); @@ -559,7 +558,7 @@ public class YARNRunner implements ClientProtocol { TaskLocationHint[] inputSplitLocations = getMapLocationHintsFromInputSplits(jobId, fs, jobConf, jobSubmitDir); - Resource mapResource = BuilderUtils.newResource( + Resource mapResource = Resource.newInstance( jobConf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB), jobConf.getInt(MRJobConfig.MAP_CPU_VCORES, @@ -602,7 +601,7 @@ public class YARNRunner implements ClientProtocol { Map<String, String> reduceEnv = new HashMap<String, String>(); setupMapReduceEnv(jobConf, reduceEnv, false); - Resource reduceResource = BuilderUtils.newResource( + Resource reduceResource = Resource.newInstance( jobConf.getInt(MRJobConfig.REDUCE_MEMORY_MB, MRJobConfig.DEFAULT_REDUCE_MEMORY_MB), jobConf.getInt(MRJobConfig.REDUCE_CPU_VCORES,
