Author: cdouglas
Date: Wed May 14 17:51:40 2014
New Revision: 1594656
URL: http://svn.apache.org/r1594656
Log:
MAPREDUCE-5821. Avoid unintentional reallocation of byte arrays in segments
during merge. Contributed by Todd Lipcon
Modified:
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java
Modified: hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt?rev=1594656&r1=1594655&r2=1594656&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt Wed
May 14 17:51:40 2014
@@ -48,6 +48,9 @@ Release 2.4.1 - UNRELEASED
MAPREDUCE-5835. Killing Task might cause the job to go to ERROR state
(Ming Ma via jlowe)
+ MAPREDUCE-5821. Avoid unintentional reallocation of byte arrays in segments
+ during merge. (Todd Lipcon via cdouglas)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java?rev=1594656&r1=1594655&r2=1594656&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java
Wed May 14 17:51:40 2014
@@ -537,6 +537,8 @@ public class Merger {
}
}
minSegment = top();
+ long startPos = minSegment.getPosition();
+ key = minSegment.getKey();
if (!minSegment.inMemory()) {
//When we load the value from an inmemory segment, we reset
//the "value" DIB in this class to the inmem segment's byte[].
@@ -547,11 +549,11 @@ public class Merger {
//segment, we reset the "value" DIB to the byte[] in that (so
//we reuse the disk segment DIB whenever we consider
//a disk segment).
+ minSegment.getValue(diskIFileValue);
value.reset(diskIFileValue.getData(), diskIFileValue.getLength());
+ } else {
+ minSegment.getValue(value);
}
- long startPos = minSegment.getPosition();
- key = minSegment.getKey();
- minSegment.getValue(value);
long endPos = minSegment.getPosition();
totalBytesProcessed += endPos - startPos;
mergeProgress.set(totalBytesProcessed * progPerByte);
