Author: jing9
Date: Thu Jan 30 23:53:23 2014
New Revision: 1563019
URL: http://svn.apache.org/r1563019
Log:
HADOOP-10295. Allow distcp to automatically identify the checksum type of
source files and use it for the target. Contributed by Jing Zhao and Laurent
Goujon.
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1563019&r1=1563018&r2=1563019&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Jan
30 23:53:23 2014
@@ -304,6 +304,9 @@ Release 2.4.0 - UNRELEASED
HADOOP-10139. Update and improve the Single Cluster Setup document.
(Akira Ajisaka via Arpit Agarwal)
+ HADOOP-10295. Allow distcp to automatically identify the checksum type of
+ source files and use it for the target. (jing9 and Laurent Goujon)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java?rev=1563019&r1=1563018&r2=1563019&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java
Thu Jan 30 23:53:23 2014
@@ -21,21 +21,26 @@ import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.io.Writable;
/** An abstract class representing file checksums for files. */
@InterfaceAudience.Public
@InterfaceStability.Stable
public abstract class FileChecksum implements Writable {
- /** The checksum algorithm name */
+ /** The checksum algorithm name */
public abstract String getAlgorithmName();
- /** The length of the checksum in bytes */
+ /** The length of the checksum in bytes */
public abstract int getLength();
- /** The value of the checksum in bytes */
+ /** The value of the checksum in bytes */
public abstract byte[] getBytes();
+ public ChecksumOpt getChecksumOpt() {
+ return null;
+ }
+
/** Return true if both the algorithms and the values are the same. */
@Override
public boolean equals(Object other) {
@@ -50,7 +55,7 @@ public abstract class FileChecksum imple
return this.getAlgorithmName().equals(that.getAlgorithmName())
&& Arrays.equals(this.getBytes(), that.getBytes());
}
-
+
@Override
public int hashCode() {
return getAlgorithmName().hashCode() ^ Arrays.hashCode(getBytes());
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java?rev=1563019&r1=1563018&r2=1563019&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
Thu Jan 30 23:53:23 2014
@@ -56,7 +56,7 @@ public class MD5MD5CRC32FileChecksum ext
this.crcPerBlock = crcPerBlock;
this.md5 = md5;
}
-
+
@Override
public String getAlgorithmName() {
return "MD5-of-" + crcPerBlock + "MD5-of-" + bytesPerCRC +
@@ -73,10 +73,10 @@ public class MD5MD5CRC32FileChecksum ext
throw new IOException("Unknown checksum type in " + algorithm);
}
-
+
@Override
public int getLength() {return LENGTH;}
-
+
@Override
public byte[] getBytes() {
return WritableUtils.toByteArray(this);
@@ -88,6 +88,7 @@ public class MD5MD5CRC32FileChecksum ext
return DataChecksum.Type.CRC32;
}
+ @Override
public ChecksumOpt getChecksumOpt() {
return new ChecksumOpt(getCrcType(), bytesPerCRC);
}
@@ -98,12 +99,12 @@ public class MD5MD5CRC32FileChecksum ext
crcPerBlock = in.readLong();
md5 = MD5Hash.read(in);
}
-
+
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(bytesPerCRC);
out.writeLong(crcPerBlock);
- md5.write(out);
+ md5.write(out);
}
/** Write that object to xml output. */
@@ -157,11 +158,11 @@ public class MD5MD5CRC32FileChecksum ext
}
} catch (Exception e) {
throw new SAXException("Invalid attributes: bytesPerCRC=" + bytesPerCRC
- + ", crcPerBlock=" + crcPerBlock + ", crcType=" + crcType
+ + ", crcPerBlock=" + crcPerBlock + ", crcType=" + crcType
+ ", md5=" + md5, e);
}
}
-
+
@Override
public String toString() {
return getAlgorithmName() + ":" + md5;
