TEZ-3596. Number of Empty DME logged for Composite fetch is too high (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/bc3c17a5 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/bc3c17a5 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/bc3c17a5 Branch: refs/heads/master Commit: bc3c17a57ec99697ce08a334c2341b3fcaacf386 Parents: 3a1f2c6 Author: Jonathan Eagles <[email protected]> Authored: Tue Jan 31 18:04:30 2017 -0600 Committer: Jonathan Eagles <[email protected]> Committed: Tue Jan 31 18:04:30 2017 -0600 ---------------------------------------------------------------------- TEZ-3334-CHANGES.txt | 1 + .../common/shuffle/impl/ShuffleInputEventHandlerImpl.java | 2 +- .../orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/bc3c17a5/TEZ-3334-CHANGES.txt ---------------------------------------------------------------------- diff --git a/TEZ-3334-CHANGES.txt b/TEZ-3334-CHANGES.txt index 8b55322..6c9a858 100644 --- a/TEZ-3334-CHANGES.txt +++ b/TEZ-3334-CHANGES.txt @@ -4,6 +4,7 @@ Apache Tez Change Log INCOMPATIBLE CHANGES: ALL CHANGES: + TEZ-3596. Number of Empty DME logged for Composite fetch is too high TEZ-3597. Composite Fetch hangs on certain DME empty events. TEZ-3595. Composite Fetch account error for disk direct TEZ-3590. Remove google.protobuf from the tez-auxservices shaded jar http://git-wip-us.apache.org/repos/asf/tez/blob/bc3c17a5/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java index a80d21b..ee06de5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java @@ -180,7 +180,7 @@ public class ShuffleInputEventHandlerImpl implements ShuffleEventHandler { LOG.debug("Source partition: " + srcIndex + " did not generate any data. SrcAttempt: [" + srcAttemptIdentifier + "]. Not fetching."); } - numDmeEventsNoData.incrementAndGet(); + numDmeEventsNoData.getAndIncrement(); shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier.expand(0)); return; } http://git-wip-us.apache.org/repos/asf/tez/blob/bc3c17a5/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java index d34fb5f..cc24db9 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java @@ -161,7 +161,7 @@ public class ShuffleInputEventHandlerOrderedGrouped implements ShuffleEventHandl "Source partition: " + partitionId + " did not generate any data. SrcAttempt: [" + srcAttemptIdentifier + "]. Not fetching."); } - numDmeEventsNoData.incrementAndGet(); + numDmeEventsNoData.getAndIncrement(); scheduler.copySucceeded(srcAttemptIdentifier.expand(0), null, 0, 0, 0, null, true); return; } @@ -194,7 +194,7 @@ public class ShuffleInputEventHandlerOrderedGrouped implements ShuffleEventHandl LOG.debug("Source partition: " + partitionId + " did not generate any data. SrcAttempt: [" + compositeInputAttemptIdentifier + "]. Not fetching."); } - numDmeEventsNoData.addAndGet(crdmEvent.getCount()); + numDmeEventsNoData.getAndIncrement(); scheduler.copySucceeded(compositeInputAttemptIdentifier.expand(i), null, 0, 0, 0, null, true); } }
