Repository: tez Updated Branches: refs/heads/TEZ-3334 29d6df2c2 -> de2e0f268
TEZ-3606. Fix debug log for empty partitions to the expanded partitionId in the Composite case (Kuhu Shukla via jeagles) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/de2e0f26 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/de2e0f26 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/de2e0f26 Branch: refs/heads/TEZ-3334 Commit: de2e0f268750c147e583eb62d1e8ae5606590cd2 Parents: 29d6df2 Author: Jonathan Eagles <[email protected]> Authored: Fri Feb 3 15:05:14 2017 -0600 Committer: Jonathan Eagles <[email protected]> Committed: Fri Feb 3 15:05:14 2017 -0600 ---------------------------------------------------------------------- TEZ-3334-CHANGES.txt | 1 + .../common/shuffle/impl/ShuffleInputEventHandlerImpl.java | 7 ++++--- .../ShuffleInputEventHandlerOrderedGrouped.java | 7 ++++--- 3 files changed, 9 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/de2e0f26/TEZ-3334-CHANGES.txt ---------------------------------------------------------------------- diff --git a/TEZ-3334-CHANGES.txt b/TEZ-3334-CHANGES.txt index 49da5aa..1ba8ca6 100644 --- a/TEZ-3334-CHANGES.txt +++ b/TEZ-3334-CHANGES.txt @@ -4,6 +4,7 @@ Apache Tez Change Log INCOMPATIBLE CHANGES: ALL CHANGES: + TEZ-3606. Fix debug log for empty partitions to the expanded partitionId in the Composite case TEZ-3604. Remove the compositeInputAttemptIdentifier from remaining list upon fetch completion in the Ordered case TEZ-3599. Unordered Fetcher can hang if empty partitions are present TEZ-3596. Number of Empty DME logged for Composite fetch is too high http://git-wip-us.apache.org/repos/asf/tez/blob/de2e0f26/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java index ee06de5..542ec34 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java @@ -201,7 +201,7 @@ public class ShuffleInputEventHandlerImpl implements ShuffleEventHandler { } if (shufflePayload.hasEmptyPartitions()) { - CompositeInputAttemptIdentifier srcAttemptIdentifier = + CompositeInputAttemptIdentifier compositeInputAttemptIdentifier = constructInputAttemptIdentifier(crdme.getTargetIndex(), crdme.getCount(), crdme.getVersion(), shufflePayload, false); boolean allPartitionsEmpty = true; @@ -209,12 +209,13 @@ public class ShuffleInputEventHandlerImpl implements ShuffleEventHandler { int srcPartitionId = partitionId + i; allPartitionsEmpty &= emptyPartitionsBitSet.get(srcPartitionId); if (emptyPartitionsBitSet.get(srcPartitionId)) { + InputAttemptIdentifier srcAttemptIdentifier = compositeInputAttemptIdentifier.expand(i); if (LOG.isDebugEnabled()) { - LOG.debug("Source partition: " + partitionId + " did not generate any data. SrcAttempt: [" + LOG.debug("Source partition: " + srcPartitionId + " did not generate any data. SrcAttempt: [" + srcAttemptIdentifier + "]. Not fetching."); } numDmeEventsNoData.getAndIncrement(); - shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier.expand(i)); + shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier); } } http://git-wip-us.apache.org/repos/asf/tez/blob/de2e0f26/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java index cc24db9..fda899f 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java @@ -190,12 +190,13 @@ public class ShuffleInputEventHandlerOrderedGrouped implements ShuffleEventHandl int srcPartitionId = partitionId + i; allPartitionsEmpty &= emptyPartitionsBitSet.get(srcPartitionId); if (emptyPartitionsBitSet.get(srcPartitionId)) { + InputAttemptIdentifier srcInputAttemptIdentifier = compositeInputAttemptIdentifier.expand(i); if (LOG.isDebugEnabled()) { - LOG.debug("Source partition: " + partitionId + " did not generate any data. SrcAttempt: [" - + compositeInputAttemptIdentifier + "]. Not fetching."); + LOG.debug("Source partition: " + srcPartitionId + " did not generate any data. SrcAttempt: [" + + srcInputAttemptIdentifier + "]. Not fetching."); } numDmeEventsNoData.getAndIncrement(); - scheduler.copySucceeded(compositeInputAttemptIdentifier.expand(i), null, 0, 0, 0, null, true); + scheduler.copySucceeded(srcInputAttemptIdentifier, null, 0, 0, 0, null, true); } }
