Repository: tez Updated Branches: refs/heads/master 67609af4b -> ec7b481b6
TEZ-2352. Move getTaskStatistics into the RuntimeTask class. (sseth) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ec7b481b Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ec7b481b Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ec7b481b Branch: refs/heads/master Commit: ec7b481b691504677b00e9461911917e8cd2f791 Parents: 67609af Author: Siddharth Seth <[email protected]> Authored: Thu Apr 23 12:25:44 2015 -0700 Committer: Siddharth Seth <[email protected]> Committed: Thu Apr 23 12:25:44 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/tez/runtime/LogicalIOProcessorRuntimeTask.java | 8 -------- .../src/main/java/org/apache/tez/runtime/RuntimeTask.java | 7 +++++++ .../main/java/org/apache/tez/runtime/task/TaskReporter.java | 9 ++++----- 4 files changed, 12 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/ec7b481b/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 1c41714..43767d2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly ALL CHANGES: + TEZ-2352. Move getTaskStatistics into the RuntimeTask class. TEZ-2357. Tez UI: misc.js.orig is committed by accident TEZ-2261. Should add diagnostics in DAGAppMaster when recovery error happens TEZ-2340. TestRecoveryParser fails http://git-wip-us.apache.org/repos/asf/tez/blob/ec7b481b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java index 0b048da..80c2717 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java @@ -73,7 +73,6 @@ import org.apache.tez.runtime.api.impl.GroupInputSpec; import org.apache.tez.runtime.api.impl.InputSpec; import org.apache.tez.runtime.api.impl.OutputSpec; import org.apache.tez.runtime.api.impl.TaskSpec; -import org.apache.tez.runtime.api.impl.TaskStatistics; import org.apache.tez.runtime.api.impl.TezEvent; import org.apache.tez.runtime.api.impl.TezInputContextImpl; import org.apache.tez.runtime.api.impl.TezMergedInputContextImpl; @@ -118,8 +117,6 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { private final LinkedHashMap<String, LogicalInput> runInputMap; private final LinkedHashMap<String, LogicalOutput> runOutputMap; - private final TaskStatistics statistics; - private final Map<String, ByteBuffer> serviceConsumerMetadata; private final Map<String, String> envMap; @@ -182,7 +179,6 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { this.objectRegistry = objectRegistry; this.ExecutionContext = ExecutionContext; this.memAvailable = memAvailable; - this.statistics = new TaskStatistics(); } /** @@ -325,10 +321,6 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { startRouterThread(); } - - public TaskStatistics getTaskStatistics() { - return statistics; - } public void run() throws Exception { Preconditions.checkState(this.state.get() == State.INITED, http://git-wip-us.apache.org/repos/asf/tez/blob/ec7b481b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java index 4777b71..745b10b 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.runtime.api.impl.TaskSpec; +import org.apache.tez.runtime.api.impl.TaskStatistics; import org.apache.tez.runtime.api.impl.TezEvent; import org.apache.tez.runtime.api.impl.TezUmbilical; import org.apache.tez.runtime.metrics.TaskCounterUpdater; @@ -49,6 +50,7 @@ public abstract class RuntimeTask { protected final AtomicInteger eventCounter; private final AtomicBoolean taskDone; private final TaskCounterUpdater counterUpdater; + private final TaskStatistics statistics; protected RuntimeTask(TaskSpec taskSpec, Configuration tezConf, TezUmbilical tezUmbilical, String pid) { @@ -59,6 +61,7 @@ public abstract class RuntimeTask { this.eventCounter = new AtomicInteger(0); this.progress = 0.0f; this.taskDone = new AtomicBoolean(false); + this.statistics = new TaskStatistics(); this.counterUpdater = new TaskCounterUpdater(tezCounters, tezConf, pid); } @@ -109,6 +112,10 @@ public abstract class RuntimeTask { return fullCounters; } + public TaskStatistics getTaskStatistics() { + return statistics; + } + public TezTaskAttemptID getTaskAttemptID() { return taskSpec.getTaskAttemptID(); } http://git-wip-us.apache.org/repos/asf/tez/blob/ec7b481b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java index 48be8bd..b9e7217 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java @@ -34,10 +34,9 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.tez.common.TezTaskUmbilicalProtocol; -import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.records.TezTaskAttemptID; -import org.apache.tez.runtime.LogicalIOProcessorRuntimeTask; +import org.apache.tez.runtime.RuntimeTask; import org.apache.tez.runtime.api.events.TaskAttemptCompletedEvent; import org.apache.tez.runtime.api.events.TaskAttemptFailedEvent; import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent; @@ -96,7 +95,7 @@ public class TaskReporter { /** * Register a task to be tracked. Heartbeats will be sent out for this task to fetch events, etc. */ - public synchronized void registerTask(LogicalIOProcessorRuntimeTask task, + public synchronized void registerTask(RuntimeTask task, ErrorReporter errorReporter) { currentCallable = new HeartbeatCallable(task, umbilical, pollInterval, sendCounterInterval, maxEventsToGet, requestCounter, containerIdStr); @@ -123,7 +122,7 @@ public class TaskReporter { private static final int LOG_COUNTER_START_INTERVAL = 5000; // 5 seconds private static final float LOG_COUNTER_BACKOFF = 1.3f; - private final LogicalIOProcessorRuntimeTask task; + private final RuntimeTask task; private EventMetaData updateEventMetadata; private final TezTaskUmbilicalProtocol umbilical; @@ -151,7 +150,7 @@ public class TaskReporter { */ private int prevCounterSendHeartbeatNum = 0; - public HeartbeatCallable(LogicalIOProcessorRuntimeTask task, + public HeartbeatCallable(RuntimeTask task, TezTaskUmbilicalProtocol umbilical, long amPollInterval, long sendCounterInterval, int maxEventsToGet, AtomicLong requestCounter, String containerIdStr) {
