Repository: tez Updated Branches: refs/heads/master 85bd772c4 -> 7f936f614
TEZ-3907. Improve log message to include the location the writers decide to spill output (Kuhu Shukla via jlowe) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/7f936f61 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/7f936f61 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/7f936f61 Branch: refs/heads/master Commit: 7f936f614914a456e3ace9ca5a5f0d218e7c7e01 Parents: 85bd772 Author: Jason Lowe <[email protected]> Authored: Mon Mar 26 14:49:12 2018 -0500 Committer: Jason Lowe <[email protected]> Committed: Mon Mar 26 14:49:12 2018 -0500 ---------------------------------------------------------------------- .../tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java | 3 ++- .../library/common/writers/UnorderedPartitionedKVWriter.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/7f936f61/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java index cfcbd56..557a538 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java @@ -977,7 +977,8 @@ public final class DefaultSorter extends ExternalSorter implements IndexedSortab totalIndexCacheMemory += spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH; } - LOG.info(outputContext.getDestinationVertexName() + ": " + "Finished spill " + numSpills); + LOG.info(outputContext.getDestinationVertexName() + ": " + "Finished spill " + numSpills + + " at " + filename.toString()); ++numSpills; if (!isFinalMergeEnabled()) { numShuffleChunks.setValue(numSpills); http://git-wip-us.apache.org/repos/asf/tez/blob/7f936f61/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java index b9f0edf..948417d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java @@ -590,6 +590,7 @@ public class UnorderedPartitionedKVWriter extends BaseUnorderedPartitionedKVWrit this.spillPathDetails = getSpillPathDetails(false, -1, spillNumber); this.spillIndex = spillPathDetails.spillIndex; } + LOG.info("Writing spill " + spillNumber + " to " + spillPathDetails.outputFilePath.toString()); FSDataOutputStream out = rfs.create(spillPathDetails.outputFilePath); if (!SPILL_FILE_PERMS.equals(SPILL_FILE_PERMS.applyUMask(FsPermission.getUMask(conf)))) { rfs.setPermission(spillPathDetails.outputFilePath, SPILL_FILE_PERMS);
