Repository: tez Updated Branches: refs/heads/master 0306928f7 -> c38d3127a
TEZ-1355. Read host and shuffle meta-information for events only if data is generated by Outputs. Contributed by Jonathan Eagles. Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c38d3127 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c38d3127 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c38d3127 Branch: refs/heads/master Commit: c38d3127ac3c2335fa7961e7a2395e4c5aba72d8 Parents: 0306928 Author: Siddharth Seth <[email protected]> Authored: Fri Aug 1 14:24:16 2014 -0700 Committer: Siddharth Seth <[email protected]> Committed: Fri Aug 1 14:24:16 2014 -0700 ---------------------------------------------------------------------- .../tez/runtime/library/output/OnFileSortedOutput.java | 11 +++++------ .../runtime/library/output/OnFileUnorderedKVOutput.java | 10 +++++----- 2 files changed, 10 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/c38d3127/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java index 8536746..6aa319b 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java @@ -149,12 +149,6 @@ public class OnFileSortedOutput extends AbstractLogicalOutput { } protected List<Event> generateEventsOnClose() throws IOException { - String host = System.getenv(ApplicationConstants.Environment.NM_HOST - .toString()); - ByteBuffer shuffleMetadata = getContext() - .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID); - int shufflePort = ShuffleUtils.deserializeShuffleProviderMetaData(shuffleMetadata); - DataMovementEventPayloadProto.Builder payloadBuilder = DataMovementEventPayloadProto .newBuilder(); @@ -182,6 +176,11 @@ public class OnFileSortedOutput extends AbstractLogicalOutput { } } if (!sendEmptyPartitionDetails || outputGenerated) { + String host = System.getenv(ApplicationConstants.Environment.NM_HOST + .toString()); + ByteBuffer shuffleMetadata = getContext() + .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID); + int shufflePort = ShuffleUtils.deserializeShuffleProviderMetaData(shuffleMetadata); payloadBuilder.setHost(host); payloadBuilder.setPort(shufflePort); payloadBuilder.setPathComponent(getContext().getUniqueIdentifier()); http://git-wip-us.apache.org/repos/asf/tez/blob/c38d3127/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java index 32592bb..2636c5c 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java @@ -127,11 +127,6 @@ public class OnFileUnorderedKVOutput extends AbstractLogicalOutput { payloadBuilder.setData(dataProtoBuilder.build()); } - String host = getHost(); - ByteBuffer shuffleMetadata = getContext() - .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID); - int shufflePort = ShuffleUtils - .deserializeShuffleProviderMetaData(shuffleMetadata); // Set the list of empty partitions - single partition on this case. if (!outputGenerated) { LOG.info("No output was generated"); @@ -142,6 +137,11 @@ public class OnFileUnorderedKVOutput extends AbstractLogicalOutput { payloadBuilder.setEmptyPartitions(emptyPartitionsBytesString); } if (outputGenerated) { + String host = getHost(); + ByteBuffer shuffleMetadata = getContext() + .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID); + int shufflePort = ShuffleUtils + .deserializeShuffleProviderMetaData(shuffleMetadata); payloadBuilder.setHost(host); payloadBuilder.setPort(shufflePort); payloadBuilder.setPathComponent(getContext().getUniqueIdentifier());
