Author: vinodkv
Date: Thu May 30 23:54:06 2013
New Revision: 1488061
URL: http://svn.apache.org/r1488061
Log:
MAPREDUCE-5228. Bring back FileInputFormat.Counter and FileOuputFormat.Counter
for binary compatibility with 1.x mapred APIs. Contributed by Mayank Bansal.
svn merge --ignore-ancestry -c 1488060 ../../trunk/
Added:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat_Counter.properties
- copied unchanged from r1488060,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat_Counter.properties
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat_Counter.properties
- copied unchanged from r1488060,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat_Counter.properties
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat_Counter.properties
- copied unchanged from r1488060,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat_Counter.properties
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileOutputFormat_Counter.properties
- copied unchanged from r1488060,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileOutputFormat_Counter.properties
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCounters.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1488061&r1=1488060&r2=1488061&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Thu
May 30 23:54:06 2013
@@ -108,6 +108,10 @@ Release 2.0.5-beta - UNRELEASED
filecache.DistributedCache for binary compatibility with mapred in 1.x.
(Zhijie Shen via vinodkv)
+ MAPREDUCE-5228. Bring back FileInputFormat.Counter and
+ FileOuputFormat.Counter for binary compatibility with 1.x mapred APIs.
+ (Mayank Bansal via vinodkv)
+
OPTIMIZATIONS
MAPREDUCE-4974. Optimising the LineRecordReader initialize() method
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java?rev=1488061&r1=1488060&r2=1488061&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java
Thu May 30 23:54:06 2013
@@ -26,6 +26,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.text.ParseException;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
import org.apache.commons.collections.IteratorUtils;
@@ -40,10 +41,9 @@ import org.apache.hadoop.mapreduce.count
import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup;
import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup;
import org.apache.hadoop.mapreduce.counters.GenericCounter;
-import org.apache.hadoop.mapreduce.counters.LimitExceededException;
import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
-import org.apache.hadoop.mapreduce.util.CountersStrings;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter;
import com.google.common.collect.Iterators;
@@ -64,6 +64,12 @@ public class Counters
public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX;
public static int MAX_GROUP_LIMIT = Limits.GROUPS_MAX;
+ private static HashMap<String, String> depricatedCounterMap =
+ new HashMap<String, String>();
+
+ static {
+ initDepricatedMap();
+ }
public Counters() {
super(groupFactory);
@@ -73,6 +79,27 @@ public class Counters
super(newCounters, groupFactory);
}
+ @SuppressWarnings({ "deprecation" })
+ private static void initDepricatedMap() {
+ depricatedCounterMap.put(FileInputFormat.Counter.class.getName(),
+ FileInputFormatCounter.class.getName());
+ depricatedCounterMap.put(FileOutputFormat.Counter.class.getName(),
+ FileOutputFormatCounter.class.getName());
+ depricatedCounterMap.put(
+ org.apache.hadoop.mapreduce.lib.input.FileInputFormat.Counter.class
+ .getName(), FileInputFormatCounter.class.getName());
+ depricatedCounterMap.put(
+ org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.Counter.class
+ .getName(), FileOutputFormatCounter.class.getName());
+ }
+
+ private static String getNewGroupKey(String oldGroup) {
+ if (depricatedCounterMap.containsKey(oldGroup)) {
+ return depricatedCounterMap.get(oldGroup);
+ }
+ return null;
+ }
+
/**
* Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters
* @param newCounters new Counters
@@ -445,6 +472,10 @@ public class Counters
" BYTES_READ as counter name instead");
return findCounter(FileInputFormatCounter.BYTES_READ);
}
+ String newGroupKey = getNewGroupKey(group);
+ if (newGroupKey != null) {
+ group = newGroupKey;
+ }
return getGroup(group).getCounterForName(name);
}
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java?rev=1488061&r1=1488060&r2=1488061&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
Thu May 30 23:54:06 2013
@@ -61,6 +61,11 @@ public abstract class FileInputFormat<K,
public static final Log LOG =
LogFactory.getLog(FileInputFormat.class);
+
+ @Deprecated
+ public static enum Counter {
+ BYTES_READ
+ }
public static final String NUM_INPUT_FILES =
org.apache.hadoop.mapreduce.lib.input.FileInputFormat.NUM_INPUT_FILES;
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java?rev=1488061&r1=1488060&r2=1488061&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java
Thu May 30 23:54:06 2013
@@ -36,6 +36,11 @@ import org.apache.hadoop.util.Progressab
@InterfaceStability.Stable
public abstract class FileOutputFormat<K, V> implements OutputFormat<K, V> {
+ @Deprecated
+ public static enum Counter {
+ BYTES_WRITTEN
+ }
+
/**
* Set whether the output of the job is compressed.
* @param conf the {@link JobConf} to modify
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java?rev=1488061&r1=1488060&r2=1488061&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
Thu May 30 23:54:06 2013
@@ -68,6 +68,11 @@ public abstract class FileInputFormat<K,
private static final Log LOG = LogFactory.getLog(FileInputFormat.class);
private static final double SPLIT_SLOP = 1.1; // 10% slop
+
+ @Deprecated
+ public static enum Counter {
+ BYTES_READ
+ }
private static final PathFilter hiddenFileFilter = new PathFilter(){
public boolean accept(Path p){
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java?rev=1488061&r1=1488060&r2=1488061&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java
Thu May 30 23:54:06 2013
@@ -60,6 +60,11 @@ public static final String COMPRESS_CODE
public static final String COMPRESS_TYPE =
"mapreduce.output.fileoutputformat.compress.type";
public static final String OUTDIR =
"mapreduce.output.fileoutputformat.outputdir";
+ @Deprecated
+ public static enum Counter {
+ BYTES_WRITTEN
+ }
+
/**
* Set whether the output of the job is compressed.
* @param job the job to modify
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCounters.java?rev=1488061&r1=1488060&r2=1488061&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCounters.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCounters.java
Thu May 30 23:54:06 2013
@@ -80,6 +80,40 @@ public class TestJobCounters {
}
}
+ @SuppressWarnings("deprecation")
+ private void validateOldFileCounters(Counters counter, long fileBytesRead,
+ long fileBytesWritten, long mapOutputBytes,
+ long mapOutputMaterializedBytes) {
+
+ assertEquals(fileBytesRead,
+ counter.findCounter(FileInputFormat.Counter.BYTES_READ).getValue());
+
+ assertEquals(
+ fileBytesRead,
+ counter
+ .findCounter(
+
org.apache.hadoop.mapreduce.lib.input.FileInputFormat.Counter.BYTES_READ)
+ .getValue());
+
+ assertEquals(fileBytesWritten,
+ counter.findCounter(FileOutputFormat.Counter.BYTES_WRITTEN).getValue());
+
+ assertEquals(
+ fileBytesWritten,
+ counter
+ .findCounter(
+
org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.Counter.BYTES_WRITTEN)
+ .getValue());
+
+ if (mapOutputBytes >= 0) {
+ assertTrue(counter.findCounter(TaskCounter.MAP_OUTPUT_BYTES).getValue()
!= 0);
+ }
+ if (mapOutputMaterializedBytes >= 0) {
+ assertTrue(counter.findCounter(TaskCounter.MAP_OUTPUT_MATERIALIZED_BYTES)
+ .getValue() != 0);
+ }
+ }
+
private void validateCounters(Counters counter, long spillRecCnt,
long mapInputRecords, long mapOutputRecords) {
// Check if the numer of Spilled Records is same as expected
@@ -255,7 +289,7 @@ public class TestJobCounters {
// 4 records/line = 61440 output records
validateCounters(c1, 90112, 15360, 61440);
validateFileCounters(c1, inputSize, 0, 0, 0);
-
+ validateOldFileCounters(c1, inputSize, 61928, 0, 0);
}
@Test
@@ -431,6 +465,18 @@ public class TestJobCounters {
validateFileCounters(c1, inputSize, 0, -1, -1);
}
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testOldCounters() throws Exception {
+ Counters c1 = new Counters();
+ c1.incrCounter(FileInputFormat.Counter.BYTES_READ, 100);
+ c1.incrCounter(FileOutputFormat.Counter.BYTES_WRITTEN, 200);
+ c1.incrCounter(TaskCounter.MAP_OUTPUT_BYTES, 100);
+ c1.incrCounter(TaskCounter.MAP_OUTPUT_MATERIALIZED_BYTES, 100);
+ validateFileCounters(c1, 100, 200, 100, 100);
+ validateOldFileCounters(c1, 100, 200, 100, 100);
+ }
+
/**
* Increases the JVM's heap usage to the specified target value.
*/
