Author: jing9
Date: Thu Jan 9 00:53:04 2014
New Revision: 1556688
URL: http://svn.apache.org/r1556688
Log:
Merging r1554967 through r1556686 from trunk.
Added:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
- copied unchanged from r1556686,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
- copied unchanged from r1556686,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/conf/ (props
changed)
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/chain/ChainMapContextImpl.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/chain/ChainReduceContextImpl.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/reduce/WrappedReducer.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(props changed)
Propchange: hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1554967-1556686
Modified: hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/CHANGES.txt?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/CHANGES.txt Thu
Jan 9 00:53:04 2014
@@ -196,6 +196,8 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5550. Task Status message (reporter.setStatus) not shown in UI
with Hadoop 2.0 (Gera Shegalov via Sandy Ryza)
+ MAPREDUCE-3310. Custom grouping comparator cannot be set for Combiners
(tucu)
+
OPTIMIZATIONS
MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza)
@@ -267,6 +269,9 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5685. Fixed a bug with JobContext getCacheFiles API inside the
WrappedReducer class. (Yi Song via vinodkv)
+ MAPREDUCE-5689. MRAppMaster does not preempt reducers when scheduled maps
+ cannot be fulfilled. (lohit via kasha)
+
Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1554967-1556686
Propchange: hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1554967-1556686
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Thu Jan 9 00:53:04 2014
@@ -229,7 +229,8 @@ public class RMContainerAllocator extend
int completedMaps = getJob().getCompletedMaps();
int completedTasks = completedMaps + getJob().getCompletedReduces();
- if (lastCompletedTasks != completedTasks) {
+ if ((lastCompletedTasks != completedTasks) ||
+ (scheduledRequests.maps.size() > 0)) {
lastCompletedTasks = completedTasks;
recalculateReduceSchedule = true;
}
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
Thu Jan 9 00:53:04 2014
@@ -1604,6 +1604,21 @@ public class TestRMContainerAllocator {
numPendingReduces,
maxReduceRampupLimit, reduceSlowStart);
verify(allocator).rampDownReduces(anyInt());
+
+ // Test reduce ramp-down for when there are scheduled maps
+ // Since we have two scheduled Maps, rampDownReducers
+ // should be invoked twice.
+ scheduledMaps = 2;
+ assignedReduces = 2;
+ doReturn(10 * 1024).when(allocator).getMemLimit();
+ allocator.scheduleReduces(
+ totalMaps, succeededMaps,
+ scheduledMaps, scheduledReduces,
+ assignedMaps, assignedReduces,
+ mapResourceReqt, reduceResourceReqt,
+ numPendingReduces,
+ maxReduceRampupLimit, reduceSlowStart);
+ verify(allocator, times(2)).rampDownReduces(anyInt());
}
private static class RecalculateContainerAllocator extends
MyContainerAllocator {
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
Thu Jan 9 00:53:04 2014
@@ -949,12 +949,29 @@ public class JobConf extends Configurati
return get(KeyFieldBasedPartitioner.PARTITIONER_OPTIONS);
}
+ /**
+ * Get the user defined {@link WritableComparable} comparator for
+ * grouping keys of inputs to the combiner.
+ *
+ * @return comparator set by the user for grouping values.
+ * @see #setCombinerKeyGroupingComparator(Class) for details.
+ */
+ public RawComparator getCombinerKeyGroupingComparator() {
+ Class<? extends RawComparator> theClass = getClass(
+ JobContext.COMBINER_GROUP_COMPARATOR_CLASS, null, RawComparator.class);
+ if (theClass == null) {
+ return getOutputKeyComparator();
+ }
+
+ return ReflectionUtils.newInstance(theClass, this);
+ }
+
/**
* Get the user defined {@link WritableComparable} comparator for
* grouping keys of inputs to the reduce.
*
* @return comparator set by the user for grouping values.
- * @see #setOutputValueGroupingComparator(Class) for details.
+ * @see #setOutputValueGroupingComparator(Class) for details.
*/
public RawComparator getOutputValueGroupingComparator() {
Class<? extends RawComparator> theClass = getClass(
@@ -966,6 +983,37 @@ public class JobConf extends Configurati
return ReflectionUtils.newInstance(theClass, this);
}
+ /**
+ * Set the user defined {@link RawComparator} comparator for
+ * grouping keys in the input to the combiner.
+ * <p/>
+ * <p>This comparator should be provided if the equivalence rules for keys
+ * for sorting the intermediates are different from those for grouping keys
+ * before each call to
+ * {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector,
Reporter)}.</p>
+ * <p/>
+ * <p>For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed
+ * in a single call to the reduce function if K1 and K2 compare as equal.</p>
+ * <p/>
+ * <p>Since {@link #setOutputKeyComparatorClass(Class)} can be used to
control
+ * how keys are sorted, this can be used in conjunction to simulate
+ * <i>secondary sort on values</i>.</p>
+ * <p/>
+ * <p><i>Note</i>: This is not a guarantee of the combiner sort being
+ * <i>stable</i> in any sense. (In any case, with the order of available
+ * map-outputs to the combiner being non-deterministic, it wouldn't make
+ * that much sense.)</p>
+ *
+ * @param theClass the comparator class to be used for grouping keys for the
+ * combiner. It should implement <code>RawComparator</code>.
+ * @see #setOutputKeyComparatorClass(Class)
+ */
+ public void setCombinerKeyGroupingComparator(
+ Class<? extends RawComparator> theClass) {
+ setClass(JobContext.COMBINER_GROUP_COMPARATOR_CLASS,
+ theClass, RawComparator.class);
+ }
+
/**
* Set the user defined {@link RawComparator} comparator for
* grouping keys in the input to the reduce.
@@ -989,7 +1037,8 @@ public class JobConf extends Configurati
*
* @param theClass the comparator class to be used for grouping keys.
* It should implement <code>RawComparator</code>.
- * @see #setOutputKeyComparatorClass(Class)
+ * @see #setOutputKeyComparatorClass(Class)
+ * @see #setCombinerKeyGroupingComparator(Class)
*/
public void setOutputValueGroupingComparator(
Class<? extends RawComparator> theClass) {
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
Thu Jan 9 00:53:04 2014
@@ -1575,7 +1575,8 @@ abstract public class Task implements Wr
combinerClass = cls;
keyClass = (Class<K>) job.getMapOutputKeyClass();
valueClass = (Class<V>) job.getMapOutputValueClass();
- comparator = (RawComparator<K>) job.getOutputKeyComparator();
+ comparator = (RawComparator<K>)
+ job.getCombinerKeyGroupingComparator();
}
@SuppressWarnings("unchecked")
@@ -1624,7 +1625,7 @@ abstract public class Task implements Wr
this.taskId = taskId;
keyClass = (Class<K>) context.getMapOutputKeyClass();
valueClass = (Class<V>) context.getMapOutputValueClass();
- comparator = (RawComparator<K>) context.getSortComparator();
+ comparator = (RawComparator<K>)
context.getCombinerKeyGroupingComparator();
this.committer = committer;
}
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
Thu Jan 9 00:53:04 2014
@@ -949,10 +949,26 @@ public class Job extends JobContextImpl
}
/**
+ * Define the comparator that controls which keys are grouped together
+ * for a single call to combiner,
+ * {@link Reducer#reduce(Object, Iterable,
+ * org.apache.hadoop.mapreduce.Reducer.Context)}
+ *
+ * @param cls the raw comparator to use
+ * @throws IllegalStateException if the job is submitted
+ */
+ public void setCombinerKeyGroupingComparatorClass(
+ Class<? extends RawComparator> cls) throws IllegalStateException {
+ ensureState(JobState.DEFINE);
+ conf.setCombinerKeyGroupingComparator(cls);
+ }
+
+ /**
* Define the comparator that controls how the keys are sorted before they
* are passed to the {@link Reducer}.
* @param cls the raw comparator
* @throws IllegalStateException if the job is submitted
+ * @see #setCombinerKeyGroupingComparatorClass(Class)
*/
public void setSortComparatorClass(Class<? extends RawComparator> cls
) throws IllegalStateException {
@@ -967,6 +983,7 @@ public class Job extends JobContextImpl
* org.apache.hadoop.mapreduce.Reducer.Context)}
* @param cls the raw comparator to use
* @throws IllegalStateException if the job is submitted
+ * @see #setCombinerKeyGroupingComparatorClass(Class)
*/
public void setGroupingComparatorClass(Class<? extends RawComparator> cls
) throws IllegalStateException {
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java
Thu Jan 9 00:53:04 2014
@@ -167,13 +167,23 @@ public interface JobContext extends MRJo
*/
public String getJar();
- /**
- * Get the user defined {@link RawComparator} comparator for
- * grouping keys of inputs to the reduce.
- *
+ /**
+ * Get the user defined {@link RawComparator} comparator for
+ * grouping keys of inputs to the combiner.
+ *
* @return comparator set by the user for grouping values.
- * @see Job#setGroupingComparatorClass(Class) for details.
+ * @see Job#setCombinerKeyGroupingComparatorClass(Class)
*/
+ public RawComparator<?> getCombinerKeyGroupingComparator();
+
+ /**
+ * Get the user defined {@link RawComparator} comparator for
+ * grouping keys of inputs to the reduce.
+ *
+ * @return comparator set by the user for grouping values.
+ * @see Job#setGroupingComparatorClass(Class)
+ * @see #getCombinerKeyGroupingComparator()
+ */
public RawComparator<?> getGroupingComparator();
/**
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
Thu Jan 9 00:53:04 2014
@@ -93,6 +93,8 @@ public interface MRJobConfig {
public static final String KEY_COMPARATOR =
"mapreduce.job.output.key.comparator.class";
+ public static final String COMBINER_GROUP_COMPARATOR_CLASS =
"mapreduce.job.combiner.group.comparator.class";
+
public static final String GROUP_COMPARATOR_CLASS =
"mapreduce.job.output.group.comparator.class";
public static final String WORKING_DIR = "mapreduce.job.working.dir";
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/chain/ChainMapContextImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/chain/ChainMapContextImpl.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/chain/ChainMapContextImpl.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/chain/ChainMapContextImpl.java
Thu Jan 9 00:53:04 2014
@@ -167,6 +167,11 @@ class ChainMapContextImpl<KEYIN, VALUEIN
}
@Override
+ public RawComparator<?> getCombinerKeyGroupingComparator() {
+ return base.getCombinerKeyGroupingComparator();
+ }
+
+ @Override
public RawComparator<?> getGroupingComparator() {
return base.getGroupingComparator();
}
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/chain/ChainReduceContextImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/chain/ChainReduceContextImpl.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/chain/ChainReduceContextImpl.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/chain/ChainReduceContextImpl.java
Thu Jan 9 00:53:04 2014
@@ -160,6 +160,11 @@ class ChainReduceContextImpl<KEYIN, VALU
}
@Override
+ public RawComparator<?> getCombinerKeyGroupingComparator() {
+ return base.getCombinerKeyGroupingComparator();
+ }
+
+ @Override
public RawComparator<?> getGroupingComparator() {
return base.getGroupingComparator();
}
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java
Thu Jan 9 00:53:04 2014
@@ -169,6 +169,11 @@ public class WrappedMapper<KEYIN, VALUEI
}
@Override
+ public RawComparator<?> getCombinerKeyGroupingComparator() {
+ return mapContext.getCombinerKeyGroupingComparator();
+ }
+
+ @Override
public RawComparator<?> getGroupingComparator() {
return mapContext.getGroupingComparator();
}
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/reduce/WrappedReducer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/reduce/WrappedReducer.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/reduce/WrappedReducer.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/reduce/WrappedReducer.java
Thu Jan 9 00:53:04 2014
@@ -162,6 +162,11 @@ public class WrappedReducer<KEYIN, VALUE
}
@Override
+ public RawComparator<?> getCombinerKeyGroupingComparator() {
+ return reduceContext.getCombinerKeyGroupingComparator();
+ }
+
+ @Override
public RawComparator<?> getGroupingComparator() {
return reduceContext.getGroupingComparator();
}
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java
Thu Jan 9 00:53:04 2014
@@ -252,6 +252,17 @@ public class JobContextImpl implements J
return conf.getJar();
}
+ /**
+ * Get the user defined {@link RawComparator} comparator for
+ * grouping keys of inputs to the combiner.
+ *
+ * @return comparator set by the user for grouping values.
+ * @see Job#setCombinerKeyGroupingComparatorClass(Class) for details.
+ */
+ public RawComparator<?> getCombinerKeyGroupingComparator() {
+ return conf.getCombinerKeyGroupingComparator();
+ }
+
/**
* Get the user defined {@link RawComparator} comparator for
* grouping keys of inputs to the reduce.
Modified:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java?rev=1556688&r1=1556687&r2=1556688&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
Thu Jan 9 00:53:04 2014
@@ -582,7 +582,7 @@ public class MergeManagerImpl<K, V> impl
Class<K> keyClass = (Class<K>) job.getMapOutputKeyClass();
Class<V> valClass = (Class<V>) job.getMapOutputValueClass();
RawComparator<K> comparator =
- (RawComparator<K>)job.getOutputKeyComparator();
+ (RawComparator<K>)job.getCombinerKeyGroupingComparator();
try {
CombineValuesIterator values = new CombineValuesIterator(
kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
Propchange:
hadoop/common/branches/HDFS-5698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1554967-1556686
