Repository: tez Updated Branches: refs/heads/master e3ee7a63b -> ca834d4f0
TEZ-3730. Lower logging level in UnorderedPartitionedKVWriter. Contributed by Prasanth Jayachandran. Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ca834d4f Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ca834d4f Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ca834d4f Branch: refs/heads/master Commit: ca834d4f09336c6a213953dae79922adc69a62eb Parents: e3ee7a6 Author: Siddharth Seth <[email protected]> Authored: Fri May 19 10:34:38 2017 -0700 Committer: Siddharth Seth <[email protected]> Committed: Fri May 19 10:34:38 2017 -0700 ---------------------------------------------------------------------- .../common/writers/UnorderedPartitionedKVWriter.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/ca834d4f/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java index b01d00f..6a9ca82 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java @@ -420,12 +420,17 @@ public class UnorderedPartitionedKVWriter extends BaseUnorderedPartitionedKVWrit currentBuffer.reset(); } else { // Update overall stats - LOG.info(destNameTrimmed + ": " + "Moving to next buffer"); + final int filledBufferCount = filledBuffers.size(); + if (LOG.isDebugEnabled() || (filledBufferCount % 10) == 0) { + LOG.info(destNameTrimmed + ": " + "Moving to next buffer. Total filled buffers: " + filledBufferCount); + } updateGlobalStats(currentBuffer); filledBuffers.add(currentBuffer); if (filledBuffers.size() >= spillLimit) { - LOG.info(destNameTrimmed + ": triggering spill"); + if (LOG.isDebugEnabled() || (filledBufferCount % 10) == 0) { + LOG.info(destNameTrimmed + ": triggering spill"); + } pendingSpillCount.incrementAndGet(); int spillNumber = numSpills.getAndIncrement(); ListenableFuture<SpillResult> future = spillExecutor.submit(new SpillCallable(
