Author: ddas
Date: Thu May 28 12:13:27 2009
New Revision: 779581
URL: http://svn.apache.org/viewvc?rev=779581&view=rev
Log:
HADOOP-5895. Fixes computation of count of merged bytes for logging.
Contributed by Ravi Gummadi.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Merger.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=779581&r1=779580&r2=779581&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu May 28 12:13:27 2009
@@ -707,6 +707,9 @@
HADOOP-5623. Fixes a problem to do with status messages getting overwritten
in streaming jobs. (Rick Cox and Jothi Padmanabhan via ddas)
+ HADOOP-5895. Fixes computation of count of merged bytes for logging.
+ (Ravi Gummadi via ddas)
+
Release 0.20.1 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Merger.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Merger.java?rev=779581&r1=779580&r2=779581&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Merger.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Merger.java Thu May
28 12:13:27 2009
@@ -513,9 +513,9 @@
long startPos = segment.getPosition();
boolean hasNext = segment.nextRawKey();
long endPos = segment.getPosition();
- startBytes += endPos - startPos;
if (hasNext) {
+ startBytes += endPos - startPos;
segmentsToMerge.add(segment);
segmentsConsidered++;
}
@@ -691,13 +691,15 @@
segmentSizes.add(segments.get(i).getLength());
}
- if (includeFinalMerge) {
- // just increment so that the following while loop iterates
- // for 1 more iteration. This is to include final merge as part of
- // the computation of expected input bytes of merges
- n++;
- }
- while (n > f) {
+ // If includeFinalMerge is true, allow the following while loop iterate
+ // for 1 more iteration. This is to include final merge as part of the
+ // computation of expected input bytes of merges
+ boolean considerFinalMerge = includeFinalMerge;
+
+ while (n > f || considerFinalMerge) {
+ if (n <=f ) {
+ considerFinalMerge = false;
+ }
long mergedSize = 0;
f = Math.min(f, segmentSizes.size());
for (int j = 0; j < f; j++) {
