Author: omalley
Date: Fri Mar 4 03:35:59 2011
New Revision: 1077053
URL: http://svn.apache.org/viewvc?rev=1077053&view=rev
Log:
commit 5bba4d073860b1bee13434ea803d1920e9265ccc
Author: Tsz Wo Wo Sze <[email protected]>
Date: Mon Nov 16 19:35:16 2009 +0000
HADOOP:5675 from
https://issues.apache.org/jira/secure/attachment/12406687/5675_20090428.patch
+++ b/YAHOO-CHANGES.txt
+ HADOOP-5675. Do not launch a job if DistCp has no work to do. (Tsz Wo
+ (Nicholas), SZE via cdouglas)
+
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/DistCp.java
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/DistCp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/DistCp.java?rev=1077053&r1=1077052&r2=1077053&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/DistCp.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/DistCp.java
Fri Mar 4 03:35:59 2011
@@ -647,8 +647,9 @@ public class DistCp implements Tool {
//Initialize the mapper
try {
- setup(conf, job, args);
- JobClient.runJob(job);
+ if (setup(conf, job, args)) {
+ JobClient.runJob(job);
+ }
finalize(conf, job, args.dst, args.preservedAttributes);
} finally {
//delete tmp
@@ -968,8 +969,9 @@ public class DistCp implements Tool {
* @param conf : The dfs/mapred configuration.
* @param jobConf : The handle to the jobConf object to be initialized.
* @param args Arguments
+ * @return true if it is necessary to launch a job.
*/
- private static void setup(Configuration conf, JobConf jobConf,
+ private static boolean setup(Configuration conf, JobConf jobConf,
final Arguments args)
throws IOException {
jobConf.set(DST_DIR_LABEL, args.dst.toUri().toString());
@@ -1139,10 +1141,13 @@ public class DistCp implements Tool {
(dstExists && !dstIsDir) || (!dstExists && srcCount == 1)?
args.dst.getParent(): args.dst, "_distcp_tmp_" + randomId);
jobConf.set(TMP_DIR_LABEL, tmpDir.toUri().toString());
- LOG.info("srcCount=" + srcCount);
+ LOG.info("sourcePathsCount=" + srcCount);
+ LOG.info("filesToCopyCount=" + fileCount);
+ LOG.info("bytesToCopyCount=" + StringUtils.humanReadableInt(byteCount));
jobConf.setInt(SRC_COUNT_LABEL, srcCount);
jobConf.setLong(TOTAL_SIZE_LABEL, byteCount);
setMapCount(byteCount, jobConf);
+ return fileCount > 0;
}
/**
