Author: cdouglas
Date: Thu Apr 24 16:05:34 2008
New Revision: 651456
URL: http://svn.apache.org/viewvc?rev=651456&view=rev
Log:
HADOOP-3294. Fix distcp to check the destination length and retry the copy if
it doesn't match the src length. Contributed by Tsz Wo (Nicholas), SZE.
Modified:
hadoop/core/branches/branch-0.17/CHANGES.txt
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/util/CopyFiles.java
Modified: hadoop/core/branches/branch-0.17/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/CHANGES.txt?rev=651456&r1=651455&r2=651456&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.17/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.17/CHANGES.txt Thu Apr 24 16:05:34 2008
@@ -580,6 +580,9 @@
HADOOP-3285. Fix input split locality when the splits align to
fs blocks. (omalley)
+ HADOOP-3294. Fix distcp to check the destination length and retry the copy
+ if it doesn't match the src length. (Tsz Wo (Nicholas), SZE via cdouglas)
+
Release 0.16.3 - 2008-04-16
BUG FIXES
Modified:
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/util/CopyFiles.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/util/CopyFiles.java?rev=651456&r1=651455&r2=651456&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/util/CopyFiles.java
(original)
+++
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/util/CopyFiles.java
Thu Apr 24 16:05:34 2008
@@ -382,13 +382,11 @@
checkAndClose(out);
}
- final boolean success = cbcopied == srcstat.getLen();
- if (!success) {
- final String badlen = "ERROR? copied " + bytesString(cbcopied)
- + " but expected " + bytesString(srcstat.getLen())
- + " from " + srcstat.getPath();
- LOG.warn(badlen);
- outc.collect(null, new Text(badlen));
+ if (cbcopied != srcstat.getLen()) {
+ throw new IOException("File size not matched: copied "
+ + bytesString(cbcopied) + " to tmpfile (=" + tmpfile
+ + ") but expected " + bytesString(srcstat.getLen())
+ + " from " + srcstat.getPath());
}
else {
if (totfiles == 1) {
@@ -408,7 +406,16 @@
throw new IOException("Failed to craete parent dir: " +
absdst.getParent());
}
rename(tmpfile, absdst);
- updatePermissions(srcstat, destFileSys.getFileStatus(absdst));
+
+ FileStatus dststat = destFileSys.getFileStatus(absdst);
+ if (dststat.getLen() != srcstat.getLen()) {
+ destFileSys.delete(absdst, false);
+ throw new IOException("File size not matched: copied "
+ + bytesString(dststat.getLen()) + " to dst (=" + absdst
+ + ") but expected " + bytesString(srcstat.getLen())
+ + " from " + srcstat.getPath());
+ }
+ updatePermissions(srcstat, dststat);
}
// report at least once for each file
