Author: sseth
Date: Fri Jul 27 04:24:27 2012
New Revision: 1366280
URL: http://svn.apache.org/viewvc?rev=1366280&view=rev
Log:
MAPREDUCE-782. Use PureJavaCrc32 in mapreduce spills. (Todd Lipcon, backport by
Brandon Li via sseth)
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/SpillRecord.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1366280&r1=1366279&r2=1366280&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Fri Jul 27 04:24:27 2012
@@ -51,6 +51,8 @@ Release 1.2.0 - unreleased
HDFS-496. Backport: Use PureJavaCrc32 in HDFS. (Brandon Li via szetszwo)
+ MAPREDUCE-782. Use PureJavaCrc32 in mapreduce spills. (Todd Lipcon,
backport by Brandon Li via sseth)
+
OPTIMIZATIONS
HDFS-2533. Backport: Remove needless synchronization on some FSDataSet
Modified:
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/SpillRecord.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/SpillRecord.java?rev=1366280&r1=1366279&r2=1366280&view=diff
==============================================================================
---
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/SpillRecord.java
(original)
+++
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/SpillRecord.java
Fri Jul 27 04:24:27 2012
@@ -22,7 +22,6 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.LongBuffer;
-import java.util.zip.CRC32;
import java.util.zip.CheckedInputStream;
import java.util.zip.CheckedOutputStream;
import java.util.zip.Checksum;
@@ -33,6 +32,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SecureIOUtils;
+import org.apache.hadoop.util.PureJavaCrc32;
import static org.apache.hadoop.mapred.MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH;
@@ -51,7 +51,7 @@ class SpillRecord {
public SpillRecord(Path indexFileName, JobConf job, String
expectedIndexOwner)
throws IOException {
- this(indexFileName, job, new CRC32(), expectedIndexOwner);
+ this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
public SpillRecord(Path indexFileName, JobConf job, Checksum crc,
@@ -115,7 +115,7 @@ class SpillRecord {
*/
public void writeToFile(Path loc, JobConf job)
throws IOException {
- writeToFile(loc, job, new CRC32());
+ writeToFile(loc, job, new PureJavaCrc32());
}
public void writeToFile(Path loc, JobConf job, Checksum crc)
