Author: mukund
Date: Tue Apr 29 17:35:16 2008
New Revision: 652202
URL: http://svn.apache.org/viewvc?rev=652202&view=rev
Log:
HADOOP-3294. Fix distcp to check the destination length and retry the copy if
it doesn't match the src length. (Tsz Wo (Nicholas), SZE via mukund)
Modified:
hadoop/core/branches/branch-0.16/CHANGES.txt
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/util/CopyFiles.java
Modified: hadoop/core/branches/branch-0.16/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/CHANGES.txt?rev=652202&r1=652201&r2=652202&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.16/CHANGES.txt Tue Apr 29 17:35:16 2008
@@ -7,6 +7,9 @@
HADOOP-3138. DFS mkdirs() should not throw an exception if the directory
already exists. (rangadi via mukund)
+ HADOOP-3294. Fix distcp to check the destination length and retry the copy
+ if it doesn't match the src length. (Tsz Wo (Nicholas), SZE via mukund)
+
Release 0.16.3 - 2008-04-16
BUG FIXES
Modified:
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/util/CopyFiles.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/util/CopyFiles.java?rev=652202&r1=652201&r2=652202&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/util/CopyFiles.java
(original)
+++
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/util/CopyFiles.java
Tue Apr 29 17:35:16 2008
@@ -319,13 +319,11 @@
checkAndClose(out);
}
- final boolean success = cbcopied == srcstat.getLen();
- if (!success) {
- final String badlen = "ERROR? copied " + bytesString(cbcopied)
- + " but expected " + bytesString(srcstat.getLen())
- + " from " + srcstat.getPath();
- LOG.warn(badlen);
- outc.collect(null, new Text(badlen));
+ if (cbcopied != srcstat.getLen()) {
+ throw new IOException("File size not matched: copied "
+ + bytesString(cbcopied) + " to tmpfile (=" + tmpfile
+ + ") but expected " + bytesString(srcstat.getLen())
+ + " from " + srcstat.getPath());
}
else {
if (totfiles == 1) {
@@ -345,6 +343,15 @@
throw new IOException("Failed to craete parent dir: " +
absdst.getParent());
}
rename(destFileSys, tmpfile, absdst);
+
+ FileStatus dststat = destFileSys.getFileStatus(absdst);
+ if (dststat.getLen() != srcstat.getLen()) {
+ destFileSys.delete(absdst);
+ throw new IOException("File size not matched: copied "
+ + bytesString(dststat.getLen()) + " to dst (=" + absdst
+ + ") but expected " + bytesString(srcstat.getLen())
+ + " from " + srcstat.getPath());
+ }
}
// report at least once for each file
